From 34f2590bb40155c48cca6c9eb57e2284d68d607a Mon Sep 17 00:00:00 2001 From: lilong Date: Thu, 5 Jan 2023 14:42:05 +0800 Subject: [PATCH 001/304] =?UTF-8?q?=E5=87=BD=E6=95=B0GssClientInit?= =?UTF-8?q?=E5=86=85=E9=83=A8=E5=87=BA=E9=94=99=E8=BF=94=E5=9B=9E=E5=89=8D?= =?UTF-8?q?=E6=B2=A1=E6=9C=89=E9=87=8A=E6=94=BE=E9=94=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/libpq/auth.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/common/backend/libpq/auth.cpp b/src/common/backend/libpq/auth.cpp index b03c9a7f5..de2632d89 100644 --- a/src/common/backend/libpq/auth.cpp +++ b/src/common/backend/libpq/auth.cpp @@ -2228,6 +2228,7 @@ static int GssClientInit(GssConn* gss_conn) krbconfig = gs_getenv_r("MPPDB_KRB5_FILE_PATH"); if (krbconfig != NULL) { if (realpath(krbconfig, real_krbconfig) == NULL) { + (void)syscalllockRelease(&kerberos_conn_lock); return -1; } check_backend_env(real_krbconfig); From 78b41cb6947c1109cd3a02a73aecc7db63b334f0 Mon Sep 17 00:00:00 2001 From: lilong Date: Thu, 12 Jan 2023 11:25:08 +0800 Subject: [PATCH 002/304] =?UTF-8?q?=E5=9B=9E=E5=BD=92=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E7=94=A8=E4=BE=8Bcluster=E7=9A=84sql=E8=AE=BE=E8=AE=A1?= =?UTF-8?q?=E4=B8=8D=E5=90=88=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/test/regress/expected/cluster.out | 133 +++++++++++++------------- src/test/regress/sql/cluster.sql | 21 ++-- 2 files changed, 74 insertions(+), 80 deletions(-) diff --git a/src/test/regress/expected/cluster.out b/src/test/regress/expected/cluster.out index 48e6492dc..29572ec3d 100644 --- a/src/test/regress/expected/cluster.out +++ b/src/test/regress/expected/cluster.out @@ -57,41 +57,41 @@ INSERT INTO clstr_tst (b, c) VALUES (8, 'ocho'); -- This entry is needed to test that TOASTED values are copied correctly. INSERT INTO clstr_tst (b, c, d) VALUES (6, 'seis', repeat('xyzzy', 100000)); CLUSTER clstr_tst_c ON clstr_tst; -SELECT a,b,c,substring(d for 30), length(d) from clstr_tst ORDER BY a, b, c; +SELECT a,b,c,substring(d for 30), length(d) from clstr_tst; a | b | c | substring | length ----+----+---------------+--------------------------------+-------- - 1 | 11 | once | | - 2 | 10 | diez | | - 3 | 31 | treinta y uno | | - 4 | 22 | veintidos | | - 5 | 3 | tres | | - 6 | 20 | veinte | | - 7 | 23 | veintitres | | - 8 | 21 | veintiuno | | - 9 | 4 | cuatro | | 10 | 14 | catorce | | - 11 | 2 | dos | | + 18 | 5 | cinco | | + 9 | 4 | cuatro | | + 26 | 19 | diecinueve | | 12 | 18 | dieciocho | | - 13 | 27 | veintisiete | | - 14 | 25 | veinticinco | | + 30 | 16 | dieciseis | | + 24 | 17 | diecisiete | | + 2 | 10 | diez | | + 23 | 12 | doce | | + 11 | 2 | dos | | + 25 | 9 | nueve | | + 31 | 8 | ocho | | + 1 | 11 | once | | + 28 | 15 | quince | | + 32 | 6 | seis | xyzzyxyzzyxyzzyxyzzyxyzzyxyzzy | 500000 + 29 | 7 | siete | | 15 | 13 | trece | | - 16 | 28 | veintiocho | | + 22 | 30 | treinta | | 17 | 32 | treinta y dos | | - 18 | 5 | cinco | | - 19 | 29 | veintinueve | | + 3 | 31 | treinta y uno | | + 5 | 3 | tres | | 20 | 1 | uno | | + 6 | 20 | veinte | | + 14 | 25 | veinticinco | | 21 | 24 | veinticuatro | | - 22 | 30 | treinta | | - 23 | 12 | doce | | - 24 | 17 | diecisiete | | - 25 | 9 | nueve | | - 26 | 19 | diecinueve | | + 4 | 22 | veintidos | | + 19 | 29 | veintinueve | | + 16 | 28 | veintiocho | | 27 | 26 | veintiseis | | - 28 | 15 | quince | | - 29 | 7 | siete | | - 30 | 16 | dieciseis | | - 31 | 8 | ocho | | - 32 | 6 | seis | xyzzyxyzzyxyzzyxyzzyxyzzyxyzzy | 500000 + 13 | 27 | veintisiete | | + 7 | 23 | veintitres | | + 8 | 21 | veintiuno | | (32 rows) SELECT a,b,c,substring(d for 30), length(d) from clstr_tst ORDER BY a; @@ -207,41 +207,41 @@ SELECT a,b,c,substring(d for 30), length(d) from clstr_tst ORDER BY c; -- Verify that inheritance link still works -- INSERT INTO clstr_tst_inh VALUES (0, 100, 'in child table'); -SELECT a,b,c,substring(d for 30), length(d) from clstr_tst ORDER BY a, b, c; +SELECT a,b,c,substring(d for 30), length(d) from clstr_tst; a | b | c | substring | length ----+----+---------------+--------------------------------+-------- - 1 | 11 | once | | - 2 | 10 | diez | | - 3 | 31 | treinta y uno | | - 4 | 22 | veintidos | | - 5 | 3 | tres | | - 6 | 20 | veinte | | - 7 | 23 | veintitres | | - 8 | 21 | veintiuno | | - 9 | 4 | cuatro | | 10 | 14 | catorce | | - 11 | 2 | dos | | + 18 | 5 | cinco | | + 9 | 4 | cuatro | | + 26 | 19 | diecinueve | | 12 | 18 | dieciocho | | - 13 | 27 | veintisiete | | - 14 | 25 | veinticinco | | + 30 | 16 | dieciseis | | + 24 | 17 | diecisiete | | + 2 | 10 | diez | | + 23 | 12 | doce | | + 11 | 2 | dos | | + 25 | 9 | nueve | | + 31 | 8 | ocho | | + 1 | 11 | once | | + 28 | 15 | quince | | + 32 | 6 | seis | xyzzyxyzzyxyzzyxyzzyxyzzyxyzzy | 500000 + 29 | 7 | siete | | 15 | 13 | trece | | - 16 | 28 | veintiocho | | + 22 | 30 | treinta | | 17 | 32 | treinta y dos | | - 18 | 5 | cinco | | - 19 | 29 | veintinueve | | + 3 | 31 | treinta y uno | | + 5 | 3 | tres | | 20 | 1 | uno | | + 6 | 20 | veinte | | + 14 | 25 | veinticinco | | 21 | 24 | veinticuatro | | - 22 | 30 | treinta | | - 23 | 12 | doce | | - 24 | 17 | diecisiete | | - 25 | 9 | nueve | | - 26 | 19 | diecinueve | | + 4 | 22 | veintidos | | + 19 | 29 | veintinueve | | + 16 | 28 | veintiocho | | 27 | 26 | veintiseis | | - 28 | 15 | quince | | - 29 | 7 | siete | | - 30 | 16 | dieciseis | | - 31 | 8 | ocho | | - 32 | 6 | seis | xyzzyxyzzyxyzzyxyzzyxyzzyxyzzy | 500000 + 13 | 27 | veintisiete | | + 7 | 23 | veintitres | | + 8 | 21 | veintiuno | | (32 rows) -- Verify that foreign key link still works @@ -328,16 +328,15 @@ CLUSTER clstr_1_pkey ON clstr_1; CLUSTER clstr_2 USING clstr_2_pkey; SELECT * FROM clstr_1 UNION ALL SELECT * FROM clstr_2 UNION ALL - SELECT * FROM clstr_3 - ORDER BY 1; + SELECT * FROM clstr_3; a --- - 1 - 1 1 2 + 1 2 2 + 1 (6 rows) -- revert to the original state @@ -356,16 +355,15 @@ SET SESSION AUTHORIZATION clstr_user PASSWORD 'ttest@123'; CLUSTER; SELECT * FROM clstr_1 UNION ALL SELECT * FROM clstr_2 UNION ALL - SELECT * FROM clstr_3 - ORDER BY 1; + SELECT * FROM clstr_3; a --- - 1 - 1 1 2 2 + 1 2 + 1 (6 rows) -- cluster a single table using the indisclustered bit previously set @@ -373,8 +371,7 @@ DELETE FROM clstr_1; INSERT INTO clstr_1 VALUES (2); INSERT INTO clstr_1 VALUES (1); CLUSTER clstr_1; -SELECT * FROM clstr_1 -ORDER BY 1; +SELECT * FROM clstr_1; a --- 1 @@ -400,22 +397,22 @@ UPDATE clustertest SET key = 35 WHERE key = 40; UPDATE clustertest SET key = 60 WHERE key = 50; UPDATE clustertest SET key = 70 WHERE key = 60; UPDATE clustertest SET key = 80 WHERE key = 70; -SELECT * FROM clustertest ORDER BY 1; +SELECT * FROM clustertest; key ----- 20 30 + 100 35 80 - 100 (5 rows) CLUSTER clustertest_pkey ON clustertest; ERROR: CLUSTER cannot run inside a transaction block -SELECT * FROM clustertest ORDER BY 1; ---?ERROR: current transaction is aborted, commands ignored until end of transaction block.* +SELECT * FROM clustertest; +ERROR: current transaction is aborted, commands ignored until end of transaction block, firstChar[Q] COMMIT; -SELECT * FROM clustertest ORDER BY 1; +SELECT * FROM clustertest; key ----- 10 @@ -433,7 +430,7 @@ create temp table clstr_temp (col1 int primary key, col2 text); NOTICE: CREATE TABLE / PRIMARY KEY will create implicit index "clstr_temp_pkey" for table "clstr_temp" insert into clstr_temp values (2, 'two'), (1, 'one'); cluster clstr_temp using clstr_temp_pkey; -select * from clstr_temp order by 1; +select * from clstr_temp; col1 | col2 ------+------ 1 | one diff --git a/src/test/regress/sql/cluster.sql b/src/test/regress/sql/cluster.sql index 1664f2459..aeaae8c91 100644 --- a/src/test/regress/sql/cluster.sql +++ b/src/test/regress/sql/cluster.sql @@ -61,14 +61,14 @@ INSERT INTO clstr_tst (b, c, d) VALUES (6, 'seis', repeat('xyzzy', 100000)); CLUSTER clstr_tst_c ON clstr_tst; -SELECT a,b,c,substring(d for 30), length(d) from clstr_tst ORDER BY a, b, c; +SELECT a,b,c,substring(d for 30), length(d) from clstr_tst; SELECT a,b,c,substring(d for 30), length(d) from clstr_tst ORDER BY a; SELECT a,b,c,substring(d for 30), length(d) from clstr_tst ORDER BY b; SELECT a,b,c,substring(d for 30), length(d) from clstr_tst ORDER BY c; -- Verify that inheritance link still works -- INSERT INTO clstr_tst_inh VALUES (0, 100, 'in child table'); -SELECT a,b,c,substring(d for 30), length(d) from clstr_tst ORDER BY a, b, c; +SELECT a,b,c,substring(d for 30), length(d) from clstr_tst; -- Verify that foreign key link still works INSERT INTO clstr_tst (b, c) VALUES (1111, 'this should fail'); @@ -126,8 +126,7 @@ CLUSTER clstr_1_pkey ON clstr_1; CLUSTER clstr_2 USING clstr_2_pkey; SELECT * FROM clstr_1 UNION ALL SELECT * FROM clstr_2 UNION ALL - SELECT * FROM clstr_3 - ORDER BY 1; + SELECT * FROM clstr_3; -- revert to the original state DELETE FROM clstr_1; @@ -146,16 +145,14 @@ SET SESSION AUTHORIZATION clstr_user PASSWORD 'ttest@123'; CLUSTER; SELECT * FROM clstr_1 UNION ALL SELECT * FROM clstr_2 UNION ALL - SELECT * FROM clstr_3 - ORDER BY 1; + SELECT * FROM clstr_3; -- cluster a single table using the indisclustered bit previously set DELETE FROM clstr_1; INSERT INTO clstr_1 VALUES (2); INSERT INTO clstr_1 VALUES (1); CLUSTER clstr_1; -SELECT * FROM clstr_1 -ORDER BY 1; +SELECT * FROM clstr_1; -- Test MVCC-safety of cluster. There isn't much we can do to verify the -- results with a single backend... @@ -182,13 +179,13 @@ UPDATE clustertest SET key = 60 WHERE key = 50; UPDATE clustertest SET key = 70 WHERE key = 60; UPDATE clustertest SET key = 80 WHERE key = 70; -SELECT * FROM clustertest ORDER BY 1; +SELECT * FROM clustertest; CLUSTER clustertest_pkey ON clustertest; -SELECT * FROM clustertest ORDER BY 1; +SELECT * FROM clustertest; COMMIT; -SELECT * FROM clustertest ORDER BY 1; +SELECT * FROM clustertest; -- check that temp tables can be clustered -- Enforce use of COMMIT instead of 2PC for temporary objects @@ -198,7 +195,7 @@ SET SESSION AUTHORIZATION clstr_user PASSWORD 'ttest@123'; create temp table clstr_temp (col1 int primary key, col2 text); insert into clstr_temp values (2, 'two'), (1, 'one'); cluster clstr_temp using clstr_temp_pkey; -select * from clstr_temp order by 1; +select * from clstr_temp; drop table clstr_temp; -- clean up From c2029091f6d643b9ae5f1ad303eaa4c28ed88ab2 Mon Sep 17 00:00:00 2001 From: xiyanziran Date: Mon, 30 Jan 2023 00:56:36 -0500 Subject: [PATCH 003/304] =?UTF-8?q?=E5=B0=86=E6=9E=9A=E4=B8=BE=E7=B1=BB?= =?UTF-8?q?=E5=9E=8Brewrite=5Fparam=E5=92=8Csql=5Fbeta=5Fparam=E5=AE=9A?= =?UTF-8?q?=E4=B9=89=E6=94=B9=E4=B8=BA=E6=9B=B4=E5=AE=B9=E6=98=93=E7=BB=B4?= =?UTF-8?q?=E6=8A=A4=E7=9A=84=E6=A0=BC=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/include/utils/guc.h | 54 ++++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/src/include/utils/guc.h b/src/include/utils/guc.h index 7de2545d3..35052b608 100755 --- a/src/include/utils/guc.h +++ b/src/include/utils/guc.h @@ -389,37 +389,37 @@ typedef enum { typedef enum { NO_REWRITE = 0, /* not allow lazy agg and magic set rewrite*/ - LAZY_AGG = 1, /* allow lazy agg */ - MAGIC_SET = 2, /* allow query qual push */ - PARTIAL_PUSH = 4, /* allow partial push */ - SUBLINK_PULLUP_WITH_UNIQUE_CHECK = 8, /* allow pull sublink with unqiue check */ - SUBLINK_PULLUP_DISABLE_REPLICATED = 16, /* disable pull up sublink with replicated table */ - SUBLINK_PULLUP_IN_TARGETLIST = 32, /* allow pull sublink in targetlist */ - PRED_PUSH = 64, /* push predicate into subquery block */ - PRED_PUSH_NORMAL = 128, - PRED_PUSH_FORCE = 256, - SUBLINK_PULLUP_DISABLE_EXPR = 512, /* disable pull sublink in expr clause */ + LAZY_AGG = (1<<0), /* allow lazy agg */ + MAGIC_SET = (1<<1), /* allow query qual push */ + PARTIAL_PUSH = (1<<2), /* allow partial push */ + SUBLINK_PULLUP_WITH_UNIQUE_CHECK = (1<<3), /* allow pull sublink with unqiue check */ + SUBLINK_PULLUP_DISABLE_REPLICATED = (1<<4), /* disable pull up sublink with replicated table */ + SUBLINK_PULLUP_IN_TARGETLIST = (1<<5), /* allow pull sublink in targetlist */ + PRED_PUSH = (1<<6), /* push predicate into subquery block */ + PRED_PUSH_NORMAL = (1<<7), + PRED_PUSH_FORCE = (1<<8), + SUBLINK_PULLUP_DISABLE_EXPR = (1<<9), /* disable pull sublink in expr clause */ } rewrite_param; typedef enum { NO_BETA_FEATURE = 0, - SEL_SEMI_POISSON = 1, /* use poisson distribution model to calibrate semi join selectivity */ - SEL_EXPR_INSTR = 2, /* use pattern sel to calibrate instr() related base rel selectivity */ - PARAM_PATH_GEN = 4, /* Parametrized Path Generation */ - RAND_COST_OPT = 8, /* Optimizing sc_random_page_cost */ - PARAM_PATH_OPT = 16, /* Parametrized Path Optimization. */ - PAGE_EST_OPT = 32, /* More accurate (rowstored) index pages estimation */ - NO_UNIQUE_INDEX_FIRST = 64, /* use unique index first rule in path generation */ - JOIN_SEL_WITH_CAST_FUNC = 128, /* support cast function while calculating join selectivity */ - CANONICAL_PATHKEY = 256, /* Use canonicalize pathkeys directly */ - INDEX_COST_WITH_LEAF_PAGES_ONLY = 512, /* compute index cost with consideration of leaf-pages-only */ - PARTITION_OPFUSION = 1024, /* Enable partition opfusion */ - A_STYLE_COERCE = 2048, - PLPGSQL_STREAM_FETCHALL = 4096, /* fetch all tuple when has stream sql under plpgsql's for-loop */ - PREDPUSH_SAME_LEVEL = 8192, /* predpush same level */ - PARTITION_FDW_ON = 16384, /* support create foreign table on partitioned table */ - DISABLE_BITMAP_COST_WITH_LOSSY_PAGES = 32768, /* stop computing bitmap path cost with lossy pages */ - EXTRACT_PUSHDOWN_OR_CLAUSE = 65536 /* Extract restriction OR clauses. */ + SEL_SEMI_POISSON = (1<<0), /* use poisson distribution model to calibrate semi join selectivity */ + SEL_EXPR_INSTR = (1<<1), /* use pattern sel to calibrate instr() related base rel selectivity */ + PARAM_PATH_GEN = (1<<2), /* Parametrized Path Generation */ + RAND_COST_OPT = (1<<3), /* Optimizing sc_random_page_cost */ + PARAM_PATH_OPT = (1<<4), /* Parametrized Path Optimization. */ + PAGE_EST_OPT = (1<<5), /* More accurate (rowstored) index pages estimation */ + NO_UNIQUE_INDEX_FIRST = (1<<6), /* use unique index first rule in path generation */ + JOIN_SEL_WITH_CAST_FUNC = (1<<7), /* support cast function while calculating join selectivity */ + CANONICAL_PATHKEY = (1<<8), /* Use canonicalize pathkeys directly */ + INDEX_COST_WITH_LEAF_PAGES_ONLY = (1<<9), /* compute index cost with consideration of leaf-pages-only */ + PARTITION_OPFUSION = (1<<10), /* Enable partition opfusion */ + A_STYLE_COERCE = (1<<11), + PLPGSQL_STREAM_FETCHALL = (1<<12), /* fetch all tuple when has stream sql under plpgsql's for-loop */ + PREDPUSH_SAME_LEVEL = (1<<13), /* predpush same level */ + PARTITION_FDW_ON = (1<<14), /* support create foreign table on partitioned table */ + DISABLE_BITMAP_COST_WITH_LOSSY_PAGES = (1<<15), /* stop computing bitmap path cost with lossy pages */ + EXTRACT_PUSHDOWN_OR_CLAUSE = (1<<16) /* Extract restriction OR clauses. */ } sql_beta_param; typedef enum { From 35973b3fe1a684f5d1b02414445f7f9805f814be Mon Sep 17 00:00:00 2001 From: vastdata-xyzr Date: Thu, 23 Feb 2023 04:16:13 -0500 Subject: [PATCH 004/304] =?UTF-8?q?=E5=8E=BB=E6=8E=89EXPLAIN=E8=BE=93?= =?UTF-8?q?=E5=87=BAmin=20read=E6=97=B6=E5=A4=9A=E4=BD=99=E7=9A=84?= =?UTF-8?q?=E7=A9=BA=E6=A0=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/optimizer/commands/explain.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gausskernel/optimizer/commands/explain.cpp b/src/gausskernel/optimizer/commands/explain.cpp index aae2b52bf..3bb3c50ae 100755 --- a/src/gausskernel/optimizer/commands/explain.cpp +++ b/src/gausskernel/optimizer/commands/explain.cpp @@ -5545,7 +5545,7 @@ static void show_analyze_buffers(ExplainState* es, const PlanState* planstate, S if (shared_blks_read_max > 0) appendStringInfo(infostr, " max read=%ld", shared_blks_read_max); if (shared_blks_read_min > 0 && shared_blks_read_min < LONG_MAX) - appendStringInfo(infostr, " min read=%ld", shared_blks_read_min); + appendStringInfo(infostr, " min read=%ld", shared_blks_read_min); if (shared_blks_dirtied_max > 0) appendStringInfo(infostr, " max dirtied=%ld", shared_blks_dirtied_max); if (shared_blks_dirtied_min > 0 && shared_blks_dirtied_min < LONG_MAX) From 068ccc51aece97a4f9181f414a7df30af40ffc40 Mon Sep 17 00:00:00 2001 From: "changying.yue" Date: Sat, 17 Jun 2023 12:16:06 +0000 Subject: [PATCH 005/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=20SSE=E6=8C=87?= =?UTF-8?q?=E4=BB=A4=E4=B8=8E128=E4=BD=8D=E6=95=B4=E5=9E=8B=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E7=B1=BB=E5=9E=8B=E6=B2=A1=E6=9C=89=E9=85=8D=E5=90=88?= =?UTF-8?q?=E5=A5=BD=E5=AF=BC=E8=87=B4OG=20coredump=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/include/c.h | 51 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 43 insertions(+), 8 deletions(-) diff --git a/src/include/c.h b/src/include/c.h index d6dd28d5a..30fd919b3 100644 --- a/src/include/c.h +++ b/src/include/c.h @@ -384,20 +384,55 @@ typedef uint16 uint2; typedef uint32 uint4; /* - * Define signed 128 bit int type. - * Define unsigned 128 bit int type. + * 128-bit signed and unsigned integers + * There currently is only limited support for such types. + * E.g. 128bit literals and snprintf are not supported; but math is. + * Also, because we exclude such types when choosing MAXIMUM_ALIGNOF, + * it must be possible to coerce the compiler to allocate them on no + * more than MAXALIGN boundaries. */ + +#if defined(__GNUC__) || defined(__SUNPRO_C) || defined(__IBMC__) +#define pg_attribute_aligned(a) __attribute__((aligned(a))) +#endif + #ifndef ENABLE_DEFAULT_GCC #if !defined(WIN32) -typedef __int128 int128; -typedef unsigned __int128 uint128; + #if defined(pg_attribute_aligned) || ALIGNOF_PG_INT128_TYPE <= MAXIMUM_ALIGNOF + typedef __int128 int128 + #if defined(pg_attribute_aligned) + pg_attribute_aligned(MAXIMUM_ALIGNOF) + #endif + ; + + typedef unsigned __int128 uint128 + #if defined(pg_attribute_aligned) + pg_attribute_aligned(MAXIMUM_ALIGNOF) + #endif + ; + #else + ereport(ERROR, (errmsg("the compiler can't support int128 or uint128 aligned on a 8-byte boundary."))); + #endif #endif #else #ifdef __linux__ -#if __GNUC__ >= 7 -typedef __int128 int128; -typedef unsigned __int128 uint128; -#endif + #if __GNUC__ >= 7 + #if defined(pg_attribute_aligned) || ALIGNOF_PG_INT128_TYPE <= MAXIMUM_ALIGNOF + typedef __int128 int128 + #if defined(pg_attribute_aligned) + pg_attribute_aligned(MAXIMUM_ALIGNOF) + #endif + ; + + typedef unsigned __int128 uint128 + #if defined(pg_attribute_aligned) + pg_attribute_aligned(MAXIMUM_ALIGNOF) + #endif + ; + #else + ereport(ERROR, (errmsg("the compiler can't support int128 or uint128 aligned on a 8-byte boundary."))); + #endif + #endif #endif #endif From 95497d07d32090d095d02ca9a296518224b7b108 Mon Sep 17 00:00:00 2001 From: cca5507 Date: Mon, 19 Jun 2023 10:40:53 +0800 Subject: [PATCH 006/304] =?UTF-8?q?thread=20pool=20worker=E7=BA=BF?= =?UTF-8?q?=E7=A8=8B=E9=80=80=E5=87=BA=E6=97=B6=E6=B8=85=E7=90=86=E5=85=B1?= =?UTF-8?q?=E4=BA=AB=E5=86=85=E5=AD=98=E6=95=B0=E7=BB=84=E4=B8=AD=E7=9A=84?= =?UTF-8?q?=E7=9B=B8=E5=85=B3=E6=A7=BD=E4=BD=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/process/postmaster/pgstat.cpp | 74 ++++++++++++++----- 1 file changed, 54 insertions(+), 20 deletions(-) diff --git a/src/gausskernel/process/postmaster/pgstat.cpp b/src/gausskernel/process/postmaster/pgstat.cpp index 6b0b2f54a..5e21138ab 100644 --- a/src/gausskernel/process/postmaster/pgstat.cpp +++ b/src/gausskernel/process/postmaster/pgstat.cpp @@ -3302,6 +3302,33 @@ void pgstat_couple_decouple_session(bool is_couple) pgstat_increment_changecount_after(beentry); } +static void clear_backend_entry(volatile PgBackendStatus* beentry) +{ + /* + * Clear status entry, following the protocol of bumping st_changecount + * before and after. We use a volatile pointer here to ensure the + * compiler doesn't try to get cute. + */ + pgstat_increment_changecount_before(beentry); + + beentry->st_procpid = 0; /* mark pid invalid */ + beentry->st_sessionid = 0; /* mark sessionid invalid */ + beentry->globalSessionId.sessionId = 0; + beentry->globalSessionId.nodeId = 0; + beentry->globalSessionId.seq = 0; + + /* + * make sure st_changecount is an even before release it. + * + * In case some thread was interrupted by SIGTERM at any time with a mess st_changecount + * in PgBackendStatus, PgstatCollectorMain may hang-up in waiting its change to even and + * can not exit after receiving SIGTERM signal + */ + do { + pgstat_increment_changecount_after(beentry); + } while ((beentry->st_changecount & 1) != 0); +} + /* * Shut down a single backend's statistics reporting at process exit. * @@ -3328,29 +3355,18 @@ static void pgstat_beshutdown_hook(int code, Datum arg) if (OidIsValid(u_sess->proc_cxt.MyDatabaseId)) pgstat_report_stat(true); - /* - * Clear my status entry, following the protocol of bumping st_changecount - * before and after. We use a volatile pointer here to ensure the - * compiler doesn't try to get cute. - */ - pgstat_increment_changecount_before(beentry); - - beentry->st_procpid = 0; /* mark pid invalid */ - beentry->st_sessionid = 0; /* mark sessionid invalid */ - beentry->globalSessionId.sessionId = 0; - beentry->globalSessionId.nodeId = 0; - beentry->globalSessionId.seq = 0; + /* Clear my status entry */ + clear_backend_entry(beentry); /* - * make sure st_changecount is an even before release it. - * - * In case some thread was interrupted by SIGTERM at any time with a mess st_changecount - * in PgBackendStatus, PgstatCollectorMain may hang-up in waiting its change to even and - * can not exit after receiving SIGTERM signal + * Thread pool worker also needs to clear + * t_thrd.shemem_ptr_cxt.BackendStatusArray[t_thrd.proc_cxt.MyBackendId - 1] */ - do { - pgstat_increment_changecount_after(beentry); - } while ((beentry->st_changecount & 1) != 0); + if (IS_THREAD_POOL_WORKER && t_thrd.proc_cxt.MyBackendId != InvalidBackendId && + beentry != &t_thrd.shemem_ptr_cxt.BackendStatusArray[t_thrd.proc_cxt.MyBackendId - 1]) { + clear_backend_entry(&t_thrd.shemem_ptr_cxt.BackendStatusArray[t_thrd.proc_cxt.MyBackendId - 1]); + WaitUntilLWLockInfoNeverAccess(&t_thrd.shemem_ptr_cxt.BackendStatusArray[t_thrd.proc_cxt.MyBackendId - 1]); + } /* * handle below cases: @@ -8267,6 +8283,12 @@ static void endMySessionTimeEntry(int code, Datum arg) { DetachMySessionTimeEntry(t_thrd.shemem_ptr_cxt.mySessionTimeEntry); + if (IS_THREAD_POOL_WORKER && t_thrd.proc_cxt.MyBackendId != InvalidBackendId && + t_thrd.shemem_ptr_cxt.mySessionTimeEntry != + &t_thrd.shemem_ptr_cxt.sessionTimeArray[t_thrd.proc_cxt.MyBackendId - 1]) { + DetachMySessionTimeEntry(&t_thrd.shemem_ptr_cxt.sessionTimeArray[t_thrd.proc_cxt.MyBackendId - 1]); + } + /* don't bother this entry any more. */ t_thrd.shemem_ptr_cxt.mySessionTimeEntry = NULL; } @@ -8509,6 +8531,12 @@ static void endMySessionStatEntry(int code, Datum arg) /* mark my entry not active. */ t_thrd.shemem_ptr_cxt.mySessionStatEntry->isValid = false; + if (IS_THREAD_POOL_WORKER && t_thrd.proc_cxt.MyBackendId != InvalidBackendId && + t_thrd.shemem_ptr_cxt.mySessionStatEntry != + &t_thrd.shemem_ptr_cxt.sessionStatArray[t_thrd.proc_cxt.MyBackendId - 1]) { + t_thrd.shemem_ptr_cxt.sessionStatArray[t_thrd.proc_cxt.MyBackendId - 1].isValid = false; + } + /* don't bother this entry any more. */ t_thrd.shemem_ptr_cxt.mySessionStatEntry = NULL; } @@ -8744,6 +8772,12 @@ static void endMySessionMemoryEntry(int code, Datum arg) /* mark my entry not active. */ t_thrd.shemem_ptr_cxt.mySessionMemoryEntry->isValid = false; + if (IS_THREAD_POOL_WORKER && t_thrd.proc_cxt.MyBackendId != InvalidBackendId && + t_thrd.shemem_ptr_cxt.mySessionMemoryEntry != + &t_thrd.shemem_ptr_cxt.sessionMemoryArray[t_thrd.proc_cxt.MyBackendId - 1]) { + t_thrd.shemem_ptr_cxt.sessionMemoryArray[t_thrd.proc_cxt.MyBackendId - 1].isValid = false; + } + /* don't bother this entry any more. */ t_thrd.shemem_ptr_cxt.mySessionMemoryEntry = NULL; } From 0aa4df4c6dadc9bf9900953b23969b59ed6828ca Mon Sep 17 00:00:00 2001 From: zhangwh Date: Sun, 25 Jun 2023 17:57:24 +0800 Subject: [PATCH 007/304] =?UTF-8?q?=E4=BC=98=E5=8C=96SMP=E5=88=9B=E5=BB=BA?= =?UTF-8?q?=E5=B9=B6=E8=A1=8C=E7=BA=BF=E7=A8=8B=E5=A4=B1=E8=B4=A5=E6=97=B6?= =?UTF-8?q?=E7=9A=84=E5=A4=84=E7=90=86=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/process/stream/streamCore.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gausskernel/process/stream/streamCore.cpp b/src/gausskernel/process/stream/streamCore.cpp index 9736c8b7d..40903ce83 100755 --- a/src/gausskernel/process/stream/streamCore.cpp +++ b/src/gausskernel/process/stream/streamCore.cpp @@ -744,7 +744,7 @@ const List* StreamNodeGroup::getStreamPairList() void StreamNodeGroup::initStreamThread(StreamProducer* producer, uint8 smpIdentifier, StreamPair* pair) { ThreadId producerThreadId = ApplyStreamThread(producer); - if (producerThreadId != 0) { + if (producerThreadId != 0 && producerThreadId != InvalidTid) { #ifdef __aarch64__ pg_memory_barrier(); #endif From 126bc7485a8ad725acde134ef6e9c27f9c824c08 Mon Sep 17 00:00:00 2001 From: pulsar Date: Tue, 30 May 2023 16:54:59 +0800 Subject: [PATCH 008/304] =?UTF-8?q?=E5=A4=84=E7=90=86=E7=BC=BA=E9=99=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/parser/analyze.cpp | 5 ++ src/common/backend/utils/cache/typcache.cpp | 23 ++++++- src/common/backend/utils/mmgr/portalmem.cpp | 32 ++++++++++ .../process/threadpool/knl_session.cpp | 2 + src/include/knl/knl_session.h | 3 + src/include/utils/plpgsql.h | 6 ++ .../regress/sql/plpgsql_cursor_rowtype.sql | 64 +++++++++++++++++++ 7 files changed, 133 insertions(+), 2 deletions(-) diff --git a/src/common/backend/parser/analyze.cpp b/src/common/backend/parser/analyze.cpp index 1677ddf06..61534896f 100644 --- a/src/common/backend/parser/analyze.cpp +++ b/src/common/backend/parser/analyze.cpp @@ -4429,7 +4429,12 @@ static Query* transformDeclareCursorStmt(ParseState* pstate, DeclareCursorStmt* ERROR, (errcode(ERRCODE_INVALID_CURSOR_DEFINITION), errmsg("cannot specify both SCROLL and NO SCROLL"))); } + /*除WITH HOLD游标外，根据DeclareCursorName对使用的row type形成依赖*/ + if (!(stmt->options & CURSOR_OPT_HOLD)) { + u_sess->analyze_cxt.DeclareCursorName = stmt->portalname; + } result = transformStmt(pstate, stmt->query); + u_sess->analyze_cxt.DeclareCursorName = NULL; /* Grammar should not have allowed anything but SELECT */ if (!IsA(result, Query) || result->commandType != CMD_SELECT || result->utilityStmt != NULL) { diff --git a/src/common/backend/utils/cache/typcache.cpp b/src/common/backend/utils/cache/typcache.cpp index 380f5199b..48777cd5f 100644 --- a/src/common/backend/utils/cache/typcache.cpp +++ b/src/common/backend/utils/cache/typcache.cpp @@ -121,6 +121,7 @@ static void cache_record_field_properties(TypeCacheEntry* typentry); static void load_enum_cache_data(TypeCacheEntry* tcache); static EnumItem* find_enumitem(TypeCacheEnumData* enum_data, Oid arg); static int enum_oid_cmp(const void* left, const void* right); +static void record_cursor_rowtype(const char* CursorName, TypeCacheEntry* typentry); void init_type_cache() { @@ -393,8 +394,13 @@ TypeCacheEntry* lookup_type_cache(Oid type_id, int flags) /* * If it's a composite type (row type), get tupdesc if requested */ - if ((flags & TYPECACHE_TUPDESC) && typentry->tupDesc == NULL && typentry->typtype == TYPTYPE_COMPOSITE) { - load_typcache_tupdesc(typentry); + if ((flags & TYPECACHE_TUPDESC) && typentry->typtype == TYPTYPE_COMPOSITE) { + if (typentry->tupDesc == NULL) { + load_typcache_tupdesc(typentry); + } + if (u_sess->analyze_cxt.DeclareCursorName) { + record_cursor_rowtype(u_sess->analyze_cxt.DeclareCursorName, typentry); + } } /* @@ -1156,3 +1162,16 @@ static int enum_oid_cmp(const void* left, const void* right) return 0; } } + +static void record_cursor_rowtype(const char* CursorName, TypeCacheEntry* typentry) +{ + MemoryContext old = MemoryContextSwitchTo(u_sess->top_transaction_mem_cxt); + Relation rel = relation_open(typentry->typrelid,AccessShareLock); + RelationIncrementReferenceCount(rel); + relation_close(rel,AccessShareLock); + CursorRecordType* var = (CursorRecordType*)palloc(sizeof(CursorRecordType)); + var->cursor_name = pstrdup(CursorName); + var->type_oid = typentry->typrelid; + u_sess->plsql_cxt.CursorRecordTypeList = lappend(u_sess->plsql_cxt.CursorRecordTypeList,var); + (void)MemoryContextSwitchTo(old); +} \ No newline at end of file diff --git a/src/common/backend/utils/mmgr/portalmem.cpp b/src/common/backend/utils/mmgr/portalmem.cpp index 3e7df1ede..fc95ea131 100755 --- a/src/common/backend/utils/mmgr/portalmem.cpp +++ b/src/common/backend/utils/mmgr/portalmem.cpp @@ -44,6 +44,7 @@ extern void ReleaseSharedCachedPlan(CachedPlan* plan, bool useResOwner); +static void CursorRecordTypeUnbind(const char* portal_name); /* * Estimate of the maximum number of open portals a user would have, * used in initially sizing the PortalHashTable in EnablePortalManager(). @@ -568,6 +569,11 @@ void PortalDrop(Portal portal, bool isTopCommit) /* drop cached plan reference, if any */ PortalReleaseCachedPlan(portal); + /*if cursor record row type*/ + if (portal->name[0] != '\0' && u_sess->plsql_cxt.CursorRecordTypeList) { + CursorRecordTypeUnbind(portal->name); + } + /* * Release any resources still attached to the portal. There are several * cases being covered here: @@ -1396,3 +1402,29 @@ HoldPinnedPortals(bool is_rollback) } } } + +/*解除游标与row type类型的依赖关系*/ +static void CursorRecordTypeUnbind(const char* portal_name) +{ + ListCell* cell = NULL; + MemoryContext old = MemoryContextSwitchTo(u_sess->top_transaction_mem_cxt); + ResourceOwner save = t_thrd.utils_cxt.CurrentResourceOwner; + t_thrd.utils_cxt.CurrentResourceOwner = t_thrd.utils_cxt.TopTransactionResourceOwner; + List* temp_list = list_copy(u_sess->plsql_cxt.CursorRecordTypeList); + foreach(cell,temp_list) { + CursorRecordType* var = (CursorRecordType*)lfirst(cell); + if (strcmp(portal_name,var->cursor_name) == 0) { + Relation rel = relation_open(var->type_oid,AccessShareLock); + if (rel->rd_refcnt > 1) { + RelationDecrementReferenceCount(rel); + } + relation_close(rel,AccessShareLock); + u_sess->plsql_cxt.CursorRecordTypeList = list_delete_ptr(u_sess->plsql_cxt.CursorRecordTypeList,var); + pfree(var->cursor_name); + pfree(var); + } + } + list_free(temp_list); + t_thrd.utils_cxt.CurrentResourceOwner = save; + (void)MemoryContextSwitchTo(old); +} diff --git a/src/gausskernel/process/threadpool/knl_session.cpp b/src/gausskernel/process/threadpool/knl_session.cpp index 57ac1b77c..f7a14e452 100755 --- a/src/gausskernel/process/threadpool/knl_session.cpp +++ b/src/gausskernel/process/threadpool/knl_session.cpp @@ -89,6 +89,7 @@ static void knl_u_analyze_init(knl_u_analyze_context* anl_cxt) anl_cxt->autoanalyze_process = NULL; anl_cxt->autoanalyze_timeinfo = NULL; anl_cxt->vac_strategy = (BufferAccessStrategyData*)palloc0(sizeof(BufferAccessStrategyData)); + anl_cxt->DeclareCursorName = NULL; } static void knl_u_attr_init(knl_session_attr* attr) @@ -865,6 +866,7 @@ static void knl_u_plpgsql_init(knl_u_plpgsql_context* plsql_cxt) plsql_cxt->cur_exception_cxt = NULL; plsql_cxt->pragma_autonomous = false; plsql_cxt->is_insert_gs_source = false; + plsql_cxt->CursorRecordTypeList = NIL; } static void knl_u_stat_init(knl_u_stat_context* stat_cxt) diff --git a/src/include/knl/knl_session.h b/src/include/knl/knl_session.h index 3089c8a4c..2c6276358 100644 --- a/src/include/knl/knl_session.h +++ b/src/include/knl/knl_session.h @@ -304,6 +304,8 @@ typedef struct knl_u_analyze_context { struct StringInfoData* autoanalyze_timeinfo; struct BufferAccessStrategyData* vac_strategy; + + char* DeclareCursorName; } knl_u_analyze_context; #define PATH_SEED_FACTOR_LEN 3 @@ -1662,6 +1664,7 @@ typedef struct knl_u_plpgsql_context { bool pragma_autonomous; /* save autonomous flag */ char* debug_query_string; bool is_insert_gs_source; /* is doing insert gs_source? */ + List* CursorRecordTypeList; /*Save the type recorded during the cursor definition*/ } knl_u_plpgsql_context; //this is used to define functions in package diff --git a/src/include/utils/plpgsql.h b/src/include/utils/plpgsql.h index e9ad4e26a..f6c454967 100644 --- a/src/include/utils/plpgsql.h +++ b/src/include/utils/plpgsql.h @@ -1915,6 +1915,12 @@ typedef struct ExceptionContext { PLpgSQL_declare_handler handler_type; } ExceptionContext; +/*Save the type recorded during the cursor definition*/ +typedef struct CursorRecordType { + char* cursor_name; + Oid type_oid; +} CursorRecordType; + /* Quick access array state */ #define IS_ARRAY_STATE(state_list, state) ((state_list && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) ? \ (linitial_int(state_list) == state) : false) diff --git a/src/test/regress/sql/plpgsql_cursor_rowtype.sql b/src/test/regress/sql/plpgsql_cursor_rowtype.sql index 9b82a565f..7698c70e6 100644 --- a/src/test/regress/sql/plpgsql_cursor_rowtype.sql +++ b/src/test/regress/sql/plpgsql_cursor_rowtype.sql @@ -694,6 +694,70 @@ set behavior_compat_options=''; drop procedure check_compile; +--游标依赖row type，后续alter type +create type foo as (a int, b int); + +--游标依赖type，alter type报错 +begin; +declare c cursor for select (i,2^30)::foo from generate_series(1,10) i; +fetch c; +fetch c; +alter type foo alter attribute b type text;--error +end; + +--第二次开始从缓存中获取type +begin; +cursor c for select (i,2^30)::foo from generate_series(1,10) i; +fetch c; +fetch c; +alter type foo alter attribute b type text;--error +end; + +--close后，可以成功alter +begin; +declare c cursor for select (i,2^30)::foo from generate_series(1,10) i; +fetch c; +fetch c; +close c; +alter type foo alter attribute b type text;--success +declare c cursor for select (i,2^30)::foo from generate_series(1,10) i; +fetch c; +fetch c; +rollback; + +begin; +cursor c for select (i,2^30)::foo from generate_series(1,10) i; +close c; +alter type foo alter attribute b type text;--success +end; + +--多个游标依赖，只关闭一个 +begin; +cursor c1 for select (i,2^30)::foo from generate_series(1,10) i; +cursor c2 for select (i,2^30)::foo from generate_series(1,10) i; +close c1; +alter type foo alter attribute b type text;--error +end; + +--多个游标依赖，都关闭 +begin; +cursor c1 for select (i,2^30)::foo from generate_series(1,10) i; +cursor c2 for select (i,2^30)::foo from generate_series(1,10) i; +close c1; +close c2; +alter type foo alter attribute b type text;--success +end; + +--WITH HOLD游标，事务结束继续保留 +begin; +cursor c3 WITH HOLD for select (i,2^30)::foo from generate_series(1,10) i; +fetch c3; +end; +fetch c3; +alter type foo alter attribute b type text;--success +fetch c3; +close c3; + ---- clean ---- drop package pck1; drop package pck2; From 976ff14babf831bc8f53e1a4e1742da6068d5ae8 Mon Sep 17 00:00:00 2001 From: pulsar Date: Tue, 27 Jun 2023 16:50:06 +0800 Subject: [PATCH 009/304] update --- src/common/backend/utils/mmgr/portalmem.cpp | 39 +++++++++++++-------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/src/common/backend/utils/mmgr/portalmem.cpp b/src/common/backend/utils/mmgr/portalmem.cpp index fc95ea131..a21a12148 100755 --- a/src/common/backend/utils/mmgr/portalmem.cpp +++ b/src/common/backend/utils/mmgr/portalmem.cpp @@ -570,7 +570,7 @@ void PortalDrop(Portal portal, bool isTopCommit) PortalReleaseCachedPlan(portal); /*if cursor record row type*/ - if (portal->name[0] != '\0' && u_sess->plsql_cxt.CursorRecordTypeList) { + if (portal->name[0] != '\0' && u_sess->plsql_cxt.CursorRecordTypeList && IsTransactionBlock()) { CursorRecordTypeUnbind(portal->name); } @@ -1407,24 +1407,35 @@ HoldPinnedPortals(bool is_rollback) static void CursorRecordTypeUnbind(const char* portal_name) { ListCell* cell = NULL; + ListCell* pnext = NULL; MemoryContext old = MemoryContextSwitchTo(u_sess->top_transaction_mem_cxt); ResourceOwner save = t_thrd.utils_cxt.CurrentResourceOwner; - t_thrd.utils_cxt.CurrentResourceOwner = t_thrd.utils_cxt.TopTransactionResourceOwner; - List* temp_list = list_copy(u_sess->plsql_cxt.CursorRecordTypeList); - foreach(cell,temp_list) { - CursorRecordType* var = (CursorRecordType*)lfirst(cell); - if (strcmp(portal_name,var->cursor_name) == 0) { - Relation rel = relation_open(var->type_oid,AccessShareLock); - if (rel->rd_refcnt > 1) { - RelationDecrementReferenceCount(rel); + PG_TRY(); + { + t_thrd.utils_cxt.CurrentResourceOwner = t_thrd.utils_cxt.TopTransactionResourceOwner; + for (cell = list_head(u_sess->plsql_cxt.CursorRecordTypeList); cell != NULL; cell = pnext) { + pnext = lnext(cell); + CursorRecordType* var = (CursorRecordType*)lfirst(cell); + if (strcmp(portal_name,var->cursor_name) == 0) { + Relation rel = relation_open(var->type_oid,AccessShareLock); + if (rel->rd_refcnt > 1) { + RelationDecrementReferenceCount(rel); + } + relation_close(rel,AccessShareLock); + u_sess->plsql_cxt.CursorRecordTypeList = list_delete_ptr(u_sess->plsql_cxt.CursorRecordTypeList,var); + pfree(var->cursor_name); + pfree(var); } - relation_close(rel,AccessShareLock); - u_sess->plsql_cxt.CursorRecordTypeList = list_delete_ptr(u_sess->plsql_cxt.CursorRecordTypeList,var); - pfree(var->cursor_name); - pfree(var); } } - list_free(temp_list); + PG_CATCH(); + { + t_thrd.utils_cxt.CurrentResourceOwner = save; + (void)MemoryContextSwitchTo(old); + PG_RE_THROW(); + } + PG_END_TRY(); + t_thrd.utils_cxt.CurrentResourceOwner = save; (void)MemoryContextSwitchTo(old); } From 20dc634619366da4602aadb8817985eadadcfb54 Mon Sep 17 00:00:00 2001 From: vastdata-xyzr Date: Wed, 28 Jun 2023 14:26:50 +0800 Subject: [PATCH 010/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=20pg=5Fstat=5F=20?= =?UTF-8?q?=E7=B3=BB=E5=88=97=E7=BB=9F=E8=AE=A1=E5=87=BD=E6=95=B0=E6=97=A0?= =?UTF-8?q?=E6=B3=95=E8=8E=B7=E5=8F=96=E5=88=86=E5=8C=BA=E4=BF=A1=E6=81=AF?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/utils/adt/pgstatfuncs.cpp | 3 +++ .../expected/hw_partition_hash_dml.out | 20 +++++++++++++++++++ .../regress/sql/hw_partition_hash_dml.sql | 13 ++++++++++++ 3 files changed, 36 insertions(+) diff --git a/src/common/backend/utils/adt/pgstatfuncs.cpp b/src/common/backend/utils/adt/pgstatfuncs.cpp index 25231bfb0..ad68260ce 100644 --- a/src/common/backend/utils/adt/pgstatfuncs.cpp +++ b/src/common/backend/utils/adt/pgstatfuncs.cpp @@ -711,6 +711,9 @@ void pg_stat_get_stat_list(List** stat_list, uint32* statFlag_ref, Oid relid) } else if (isPartitionedObject(relid, RELKIND_INDEX, true)) { *statFlag_ref = relid; *stat_list = getPartitionObjectIdList(relid, PART_OBJ_TYPE_INDEX_PARTITION); + } else if (isPartitionObject(relid, PART_OBJ_TYPE_TABLE_PARTITION, true)) { + *statFlag_ref = partid_get_parentid(relid); + *stat_list = list_make1_oid(relid); } else { *statFlag_ref = InvalidOid; *stat_list = list_make1_oid(relid); diff --git a/src/test/regress/expected/hw_partition_hash_dml.out b/src/test/regress/expected/hw_partition_hash_dml.out index ba8e85eae..f9b8ca606 100644 --- a/src/test/regress/expected/hw_partition_hash_dml.out +++ b/src/test/regress/expected/hw_partition_hash_dml.out @@ -306,6 +306,26 @@ select * from test_index_ht order by 1; 6 | | (4 rows) +create table tab_hash(c1 number, c2 number,c3 varchar2(20)) partition by hash(c2)( +partition p1, +partition p2, +partition p3, +partition p4, +partition p5 +); +insert into tab_hash select t,t,t from generate_series(1,10) t; +analyse tab_hash; +select relname, pg_stat_get_live_tuples(oid) live_tuples from pg_partition +where parentid='tab_hash'::regclass and parttype='p' order by relname; + relname | live_tuples +---------+------------- + p1 | 3 + p2 | 2 + p3 | 2 + p4 | 2 + p5 | 1 +(5 rows) + drop table test_index_ht; drop schema fvt_other_cmd cascade; NOTICE: drop cascades to table fvt_other_cmd.idex_list_partition_table_001 diff --git a/src/test/regress/sql/hw_partition_hash_dml.sql b/src/test/regress/sql/hw_partition_hash_dml.sql index 1a3f8498f..30388abc8 100644 --- a/src/test/regress/sql/hw_partition_hash_dml.sql +++ b/src/test/regress/sql/hw_partition_hash_dml.sql @@ -169,5 +169,18 @@ set enable_seqscan = off; set enable_bitmapscan = off; explain (costs off, verbose on) select * from test_index_ht order by 1; select * from test_index_ht order by 1; + +create table tab_hash(c1 number, c2 number,c3 varchar2(20)) partition by hash(c2)( +partition p1, +partition p2, +partition p3, +partition p4, +partition p5 +); +insert into tab_hash select t,t,t from generate_series(1,10) t; +analyse tab_hash; +select relname, pg_stat_get_live_tuples(oid) live_tuples from pg_partition +where parentid='tab_hash'::regclass and parttype='p' order by relname; + drop table test_index_ht; drop schema fvt_other_cmd cascade; From c629fe3ac1c842fc391fd2e721079c4a63064819 Mon Sep 17 00:00:00 2001 From: April01xxx Date: Mon, 3 Jul 2023 11:42:22 +0800 Subject: [PATCH 011/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8DOptMemoryContext?= =?UTF-8?q?=E5=9C=A8Debug=E6=A8=A1=E5=BC=8F=E4=B8=8B=E6=9C=AA=E6=AD=A3?= =?UTF-8?q?=E7=A1=AE=E8=AE=BE=E7=BD=AE=E5=93=A8=E5=85=B5=E5=AF=BC=E8=87=B4?= =?UTF-8?q?=E5=86=85=E5=AD=98=E6=A3=80=E6=9F=A5=E5=A4=B1=E8=B4=A5=E7=9A=84?= =?UTF-8?q?=E9=97=AE=E9=A2=98=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/utils/mmgr/aset.cpp | 10 +++++++--- src/common/backend/utils/mmgr/opt_aset.cpp | 4 ++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/common/backend/utils/mmgr/aset.cpp b/src/common/backend/utils/mmgr/aset.cpp index 34ad4afaf..ece3f5f34 100644 --- a/src/common/backend/utils/mmgr/aset.cpp +++ b/src/common/backend/utils/mmgr/aset.cpp @@ -1603,10 +1603,14 @@ void GenericMemoryAllocator::AllocSetStats(MemoryContext context, int level) void AllocSetCheckPointer(void* pointer) { AllocChunkData* chunk = (AllocChunkData*)(((char*)(pointer)) - ALLOC_CHUNKHDRSZ); - AllocMagicData* magic = - (AllocMagicData*)(((char*)chunk) + ALLOC_CHUNKHDRSZ + MAXALIGN(chunk->requested_size) - ALLOC_MAGICHDRSZ); - Assert(magic->aset == chunk->aset && magic->size == chunk->size && magic->posnum == PosmagicNum); + /* For opt memory context, we use sentinel instead of magic number to protect memory overflow. */ + if (!IsOptAllocSetContext(chunk->aset)) { + AllocMagicData* magic = + (AllocMagicData*)(((char*)chunk) + ALLOC_CHUNKHDRSZ + MAXALIGN(chunk->requested_size) - ALLOC_MAGICHDRSZ); + + Assert(magic->aset == chunk->aset && magic->size == chunk->size && magic->posnum == PosmagicNum); + } } /* diff --git a/src/common/backend/utils/mmgr/opt_aset.cpp b/src/common/backend/utils/mmgr/opt_aset.cpp index 9073cf6e5..0e79d869d 100644 --- a/src/common/backend/utils/mmgr/opt_aset.cpp +++ b/src/common/backend/utils/mmgr/opt_aset.cpp @@ -864,6 +864,8 @@ static void* opt_AllocSetRealloc(MemoryContext context, void* pointer, Size alig if (oldsize >= size) { #ifdef MEMORY_CONTEXT_CHECKING chunk->requested_size = size; + if (size < oldsize) + set_sentinel(pointer, size); #endif return pointer; } @@ -914,6 +916,8 @@ static void* opt_AllocSetRealloc(MemoryContext context, void* pointer, Size alig chunk->size = chksize; #ifdef MEMORY_CONTEXT_CHECKING chunk->requested_size = size; + if (size < chunk->size) + set_sentinel(AllocChunkGetPointer(chunk), size); #endif return AllocChunkGetPointer(chunk); } From 73621ec6fa00de060501fb4f0d19fc1970621963 Mon Sep 17 00:00:00 2001 From: cc_db_dev Date: Mon, 3 Jul 2023 14:15:34 +0800 Subject: [PATCH 012/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dtrack=5Ffunctions?= =?UTF-8?q?=E5=BC=80=E5=90=AF=E4=B8=8B=E7=9A=84=E5=AE=95=E6=9C=BA=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 原因分析： u_sess->stat_cxt.pgStatFunctions使用的上下文错误。在原实现中，其使用了u_sess->stat_cxt.pgStatLocalContext作为pgStatFunctions的上下文， （1）但很多情况下，该pgStatLocalContext为空，导致使用了一个parent为空的上下文来管理pgStatFunctions的内存 （2）在缺陷的场景中（本次添加的用例），pgStatLocalContext不为空，但其生命周期为事务级别，而pgStatFunctions的生命周期为会话级别， 二者不匹配，导致宕机 修改方案： 采用会话级别的上下文管理u_sess->stat_cxt.pgStatFunctions。 --- src/gausskernel/process/postmaster/pgstat.cpp | 10 ++++- src/test/regress/expected/hw_function_p_3.out | 38 +++++++++++++++++++ src/test/regress/sql/hw_function_p_3.sql | 21 ++++++++++ 3 files changed, 68 insertions(+), 1 deletion(-) diff --git a/src/gausskernel/process/postmaster/pgstat.cpp b/src/gausskernel/process/postmaster/pgstat.cpp index 6b0b2f54a..11238de1b 100644 --- a/src/gausskernel/process/postmaster/pgstat.cpp +++ b/src/gausskernel/process/postmaster/pgstat.cpp @@ -1769,7 +1769,15 @@ void pgstat_init_function_usage(FunctionCallInfoData* fcinfo, PgStat_FunctionCal hash_ctl.keysize = sizeof(Oid); hash_ctl.entrysize = sizeof(PgStat_BackendFunctionEntry); hash_ctl.hash = oid_hash; - hash_ctl.hcxt = u_sess->stat_cxt.pgStatLocalContext; + /* + * hence the u_sess->stat_cxt.pgStatFunctions is used for whole session + * should use memory context under uess.top_mem_contxt here + */ + hash_ctl.hcxt = AllocSetContextCreate(u_sess->top_mem_cxt, + "Function stat hash", + ALLOCSET_SMALL_MINSIZE, + ALLOCSET_SMALL_INITSIZE, + ALLOCSET_SMALL_MAXSIZE); u_sess->stat_cxt.pgStatFunctions = hash_create( "Function stat entries", PGSTAT_FUNCTION_HASH_SIZE, &hash_ctl, HASH_ELEM | HASH_FUNCTION | HASH_CONTEXT); } diff --git a/src/test/regress/expected/hw_function_p_3.out b/src/test/regress/expected/hw_function_p_3.out index 5b68ff717..f63d730e3 100644 --- a/src/test/regress/expected/hw_function_p_3.out +++ b/src/test/regress/expected/hw_function_p_3.out @@ -54,5 +54,43 @@ select pg_table_size('t1'); 0 (1 row) +--test bug for track function +CREATE OR REPLACE FUNCTION test_func(a INT) RETURNS INT AS $$ +BEGIN + RETURN 10; +END; +$$ LANGUAGE plpgsql; +\c +set search_path to hw_function_p_3; +SET track_functions = 'all'; +BEGIN; +SELECT pg_stat_get_mem_mbytes_reserved(10); + pg_stat_get_mem_mbytes_reserved +--------------------------------- + GetFailed +(1 row) + +SELECT test_func(10); + test_func +----------- + 10 +(1 row) + +COMMIT; +SELECT test_func(10); + test_func +----------- + 10 +(1 row) + +SELECT test_func(10); + test_func +----------- + 10 +(1 row) + +reset track_functions; +drop function test_func; +--drop data drop schema hw_function_p_3 cascade; NOTICE: drop cascades to table t1 diff --git a/src/test/regress/sql/hw_function_p_3.sql b/src/test/regress/sql/hw_function_p_3.sql index 7c59e282a..d4f69c3d2 100644 --- a/src/test/regress/sql/hw_function_p_3.sql +++ b/src/test/regress/sql/hw_function_p_3.sql @@ -20,4 +20,25 @@ select * from pg_stat_get_backend_client_port(1); create table t1(a int, b int, c int); select pg_table_size('t1'); +--test bug for track function +CREATE OR REPLACE FUNCTION test_func(a INT) RETURNS INT AS $$ +BEGIN + RETURN 10; +END; +$$ LANGUAGE plpgsql; + +\c +set search_path to hw_function_p_3; +SET track_functions = 'all'; + +BEGIN; +SELECT pg_stat_get_mem_mbytes_reserved(10); +SELECT test_func(10); +COMMIT; +SELECT test_func(10); +SELECT test_func(10); +reset track_functions; +drop function test_func; + +--drop data drop schema hw_function_p_3 cascade; \ No newline at end of file From 06135d38cf9ad3e2f8d392ab6b8e284eb89f1412 Mon Sep 17 00:00:00 2001 From: kenxx Date: Tue, 4 Jul 2023 22:06:39 -0400 Subject: [PATCH 013/304] =?UTF-8?q?=E7=89=A9=E5=8C=96=E8=A7=86=E5=9B=BE?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E5=88=A0=E9=99=A4=E9=80=BB=E8=BE=91=E4=BF=AE?= =?UTF-8?q?=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/catalog/dependency.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/common/backend/catalog/dependency.cpp b/src/common/backend/catalog/dependency.cpp index 045f8dc37..bd62afbd7 100644 --- a/src/common/backend/catalog/dependency.cpp +++ b/src/common/backend/catalog/dependency.cpp @@ -1240,6 +1240,11 @@ static void doDeletion(const ObjectAddress* object, int flags) bool isTmpSequence = false; bool isTmpTable = false; + Oid mlogid = find_matview_mlog_table(object->objectId); + if (mlogid != 0 && !u_sess->attr.attr_sql.enable_cluster_resize) { + delete_matdep_table(mlogid); + } + if (relKind == RELKIND_INDEX || relKind == RELKIND_GLOBAL_INDEX) { bool concurrent = (((uint32)flags & PERFORM_DELETION_CONCURRENTLY) == PERFORM_DELETION_CONCURRENTLY); bool concurrent_lock_mode = (((uint32)flags & PERFORM_DELETION_CONCURRENTLY_LOCK) == PERFORM_DELETION_CONCURRENTLY_LOCK); @@ -1281,11 +1286,6 @@ static void doDeletion(const ObjectAddress* object, int flags) */ } - Oid mlogid = find_matview_mlog_table(object->objectId); - if (mlogid != 0 && !u_sess->attr.attr_sql.enable_cluster_resize) { - delete_matdep_table(mlogid); - } - #ifdef PGXC /* * Do not do extra process if this session is connected to a remote From e13a8ad0336ea0018a43209b751b7320abeece43 Mon Sep 17 00:00:00 2001 From: pulsar Date: Thu, 6 Jul 2023 10:26:38 +0800 Subject: [PATCH 014/304] add PG_TRY --- src/common/backend/parser/analyze.cpp | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/common/backend/parser/analyze.cpp b/src/common/backend/parser/analyze.cpp index 61534896f..a444c2236 100644 --- a/src/common/backend/parser/analyze.cpp +++ b/src/common/backend/parser/analyze.cpp @@ -4430,10 +4430,20 @@ static Query* transformDeclareCursorStmt(ParseState* pstate, DeclareCursorStmt* } /*除WITH HOLD游标外，根据DeclareCursorName对使用的row type形成依赖*/ - if (!(stmt->options & CURSOR_OPT_HOLD)) { - u_sess->analyze_cxt.DeclareCursorName = stmt->portalname; + PG_TRY(); + { + if (!(stmt->options & CURSOR_OPT_HOLD)) { + u_sess->analyze_cxt.DeclareCursorName = stmt->portalname; + } + result = transformStmt(pstate, stmt->query); } - result = transformStmt(pstate, stmt->query); + PG_CATCH(); + { + u_sess->analyze_cxt.DeclareCursorName = NULL; + PG_RE_THROW(); + } + PG_END_TRY(); + u_sess->analyze_cxt.DeclareCursorName = NULL; /* Grammar should not have allowed anything but SELECT */ From 21920c47e87ce8fde377d6e2b59ed6469e77fd38 Mon Sep 17 00:00:00 2001 From: gbzhangkai Date: Fri, 7 Jul 2023 11:14:50 +0800 Subject: [PATCH 015/304] =?UTF-8?q?initdb=20=E4=B8=AD=20main=E5=87=BD?= =?UTF-8?q?=E6=95=B0=E6=9C=89write=5Fstderr=E4=B8=AD=E5=8F=82=E6=95=B0?= =?UTF-8?q?=E4=B8=AA=E6=95=B0=E4=B8=8D=E4=B8=80=E8=87=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/initdb/initdb.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bin/initdb/initdb.cpp b/src/bin/initdb/initdb.cpp index 93eadcfb1..cb935ab6f 100644 --- a/src/bin/initdb/initdb.cpp +++ b/src/bin/initdb/initdb.cpp @@ -4787,7 +4787,7 @@ int main(int argc, char* argv[]) if (ss_issharedstorage) { int ret = ss_check_shareddir(vgdata, ss_nodeid, &ss_need_mkclusterdir); if (ret != 0) { - write_stderr("ERROR: %s: shared storage initdb failed because of the following error:\n"); + write_stderr("ERROR: %s: shared storage initdb failed because of the following error:\n", progname); if (ret & ERROR_INSTANCEDIR_EXISTS) { write_stderr(_("ERROR: [*]shared storage files of instance %d in the directory \"%s\" already exists\n"), ss_nodeid, vgdata); write_stderr(_("If you want to create a new shared storage instance, either remove shared storage " From 168bb07aef65ad9ff4f5879d1a0d24058db8d8f4 Mon Sep 17 00:00:00 2001 From: gentle_hu Date: Mon, 10 Jul 2023 16:10:34 +0800 Subject: [PATCH 016/304] fix partition opfusion core --- .../utils/cache/knl_localpartdefcache.cpp | 5 +- .../utils/cache/knl_localtabdefcache.cpp | 17 +----- src/common/backend/utils/cache/partcache.cpp | 58 +------------------ src/common/backend/utils/cache/relcache.cpp | 18 ------ .../opfusion/opfusion_indexonlyscan.cpp | 4 ++ .../runtime/opfusion/opfusion_indexscan.cpp | 6 +- .../runtime/opfusion/opfusion_util.cpp | 6 ++ 7 files changed, 21 insertions(+), 93 deletions(-) diff --git a/src/common/backend/utils/cache/knl_localpartdefcache.cpp b/src/common/backend/utils/cache/knl_localpartdefcache.cpp index fe56b8db8..bd94224f2 100644 --- a/src/common/backend/utils/cache/knl_localpartdefcache.cpp +++ b/src/common/backend/utils/cache/knl_localpartdefcache.cpp @@ -262,8 +262,7 @@ void LocalPartDefCache::InvalidateAll(void) if (PartitionHasReferenceCountZero(part)) { /* Delete this entry immediately */ PartitionClearPartition(part, false); - elt = DLGetHead(&bucket_entry->cc_bucket); - } else if (!list_member_ptr(rebuildList, part)) { + } else { rebuildList = lappend(rebuildList, part); } } @@ -345,7 +344,6 @@ void LocalPartDefCache::AtEOXact_PartitionCache(bool isCommit) part->pd_createSubid = InvalidSubTransactionId; } else { PartitionClearPartition(part, false); - elt = DLGetHead(&bucket_entry->cc_bucket); continue; } } @@ -390,7 +388,6 @@ void LocalPartDefCache::AtEOSubXact_PartitionCache(bool isCommit, SubTransaction part->pd_createSubid = parentSubid; else { PartitionClearPartition(part, false); - elt = DLGetHead(&bucket_entry->cc_bucket); continue; } } diff --git a/src/common/backend/utils/cache/knl_localtabdefcache.cpp b/src/common/backend/utils/cache/knl_localtabdefcache.cpp index 63f696492..28c51f9fe 100644 --- a/src/common/backend/utils/cache/knl_localtabdefcache.cpp +++ b/src/common/backend/utils/cache/knl_localtabdefcache.cpp @@ -678,7 +678,8 @@ void LocalTabDefCache::InitPhase3(void) uint32 hash_value = oid_hash((void *)&(rel->rd_id), sizeof(Oid)); InsertRelationIntoGlobal(rel, hash_value); } - elt = DLGetHead(&bucket_entry->cc_bucket); + bucket_elt = DLGetHead(m_bucket_list.GetActiveBucketList()); + elt = NULL; } } } @@ -752,7 +753,6 @@ void LocalTabDefCache::InvalidateRelationAll() /* Delete this entry immediately */ Assert(!rel->rd_isnailed); RelationClearRelation(rel, false); - elt = DLGetHead(&bucket_entry->cc_bucket); } else { /* * If it's a mapped rel, immediately update its rd_node in @@ -773,9 +773,6 @@ void LocalTabDefCache::InvalidateRelationAll() * next in no particular order; and everything else goes to the * back of rebuildList. */ - if (list_member_ptr(rebuildFirstList, rel) || list_member_ptr(rebuildList, rel)) { - continue; - } if (RelationGetRelid(rel) == RelationRelationId) rebuildFirstList = lcons(rel, rebuildFirstList); else if (RelationGetRelid(rel) == ClassOidIndexId) @@ -823,14 +820,7 @@ void LocalTabDefCache::InvalidateRelationNodeList() elt = DLGetSucc(elt); Relation rel = entry->rel; if (rel->rd_locator_info != NULL) { - Assert(!rel->rd_isnailed); - bool clear = RelationHasReferenceCountZero(rel) && - !(RelationIsIndex(rel) && rel->rd_refcnt > 0 && rel->rd_indexcxt != NULL); RelationClearRelation(rel, !RelationHasReferenceCountZero(rel)); - if (!clear) { - elt = DLGetHead(&bucket_entry->cc_bucket); - continue; - } } } } @@ -972,7 +962,6 @@ void LocalTabDefCache::AtEOXact_RelationCache(bool isCommit) rel->rd_createSubid = InvalidSubTransactionId; else if (RelationHasReferenceCountZero(rel)) { RelationClearRelation(rel, false); - elt = DLGetHead(&bucket_entry->cc_bucket); continue; } else { /* @@ -1005,7 +994,6 @@ void LocalTabDefCache::AtEOXact_RelationCache(bool isCommit) } if (rel->partMap != NULL && unlikely(rel->partMap->isDirty)) { RelationClearRelation(rel, false); - elt = DLGetHead(&bucket_entry->cc_bucket); } } } @@ -1049,7 +1037,6 @@ void LocalTabDefCache::AtEOSubXact_RelationCache(bool isCommit, SubTransactionId rel->rd_createSubid = parentSubid; else if (RelationHasReferenceCountZero(rel)) { RelationClearRelation(rel, false); - elt = DLGetHead(&bucket_entry->cc_bucket); continue; } else { /* diff --git a/src/common/backend/utils/cache/partcache.cpp b/src/common/backend/utils/cache/partcache.cpp index 89f8fba99..cb4fe44bd 100644 --- a/src/common/backend/utils/cache/partcache.cpp +++ b/src/common/backend/utils/cache/partcache.cpp @@ -867,8 +867,6 @@ void PartitionCacheInvalidate(void) if (PartitionHasReferenceCountZero(partition)) { /* Delete this entry immediately */ PartitionClearPartition(partition, false); - hash_seq_term(&status); - hash_seq_init(&status, u_sess->cache_cxt.PartitionIdCache); } else { rebuildList = lappend(rebuildList, partition); } @@ -987,8 +985,6 @@ void AtEOXact_PartitionCache(bool isCommit) partition->pd_createSubid = InvalidSubTransactionId; } else { PartitionClearPartition(partition, false); - hash_seq_term(&status); - hash_seq_init(&status, u_sess->cache_cxt.PartitionIdCache); continue; } } @@ -1042,8 +1038,6 @@ void AtEOSubXact_PartitionCache(bool isCommit, SubTransactionId mySubid, SubTran partition->pd_createSubid = parentSubid; else { PartitionClearPartition(partition, false); - hash_seq_term(&status); - hash_seq_init(&status, u_sess->cache_cxt.PartitionIdCache); continue; } } @@ -1089,19 +1083,11 @@ static void AtEOXact_PartRelCache() Assert(idhentry->reldesc->rd_refcnt == 1); /* destroy part first to release rd_refcnt */ PartitionCacheInvalidateEntry(idhentry->pd_id); - hash_seq_term(&status); - hash_seq_init(&status, u_sess->cache_cxt.PartRelCache); - PartRelCacheEntry *old_idhentry = idhentry; - while ((idhentry = (PartRelCacheEntry*)hash_seq_search(&status)) != NULL) { - if (old_idhentry == idhentry) { - break; - } - } Assert(idhentry->reldesc->rd_refcnt == 0); } RelationDestroyRelation(idhentry->reldesc, false); + idhentry->reldesc = NULL; } - pfree(idhentry); } } } @@ -1124,8 +1110,8 @@ static void DeletePartRelCacheEntry(PartRelCacheEntry *idhentry, HTAB *PartRelCa if (idhentry->reldesc != NULL) { Assert(idhentry->reldesc->rd_refcnt == 0); RelationDestroyRelation(idhentry->reldesc, false); + idhentry->reldesc = NULL; } - pfree(idhentry); } static void PartRelCachePdIdCallback(Datum arg, Oid partid) @@ -1151,44 +1137,6 @@ static void PartRelCachePdIdCallback(Datum arg, Oid partid) while ((idhentry = (PartRelCacheEntry*)hash_seq_search(&status)) != NULL) { DeletePartRelCacheEntry(idhentry, u_sess->cache_cxt.PartRelCache, &u_sess->cache_cxt.PartRelCacheNeedEOXActWork); - if (hash_search(u_sess->cache_cxt.PartRelCache, (void *)&partid, HASH_FIND, NULL) != NULL) { - hash_seq_term(&status); - hash_seq_init(&status, u_sess->cache_cxt.PartRelCache); - PartRelCacheEntry *old_idhentry = idhentry; - while ((idhentry = (PartRelCacheEntry*)hash_seq_search(&status)) != NULL) { - if (old_idhentry == idhentry) { - break; - } - } - } - } -} - -static void PartRelCacheRdIdCallback(Datum arg, Oid relid) -{ - if (u_sess->cache_cxt.PartRelCache == NULL) { - return; - } - - HASH_SEQ_STATUS status; - hash_seq_init(&status, u_sess->cache_cxt.PartRelCache); - PartRelCacheEntry *idhentry; - while ((idhentry = (PartRelCacheEntry*)hash_seq_search(&status)) != NULL) { - if (relid == InvalidOid || idhentry->reldesc == NULL || idhentry->reldesc->parentId == relid || - idhentry->reldesc->grandparentId == relid) { - DeletePartRelCacheEntry(idhentry, u_sess->cache_cxt.PartRelCache, - &u_sess->cache_cxt.PartRelCacheNeedEOXActWork); - if (hash_search(u_sess->cache_cxt.PartRelCache, (void *)&idhentry->pd_id, HASH_FIND, NULL) != NULL) { - hash_seq_term(&status); - hash_seq_init(&status, u_sess->cache_cxt.PartRelCache); - PartRelCacheEntry *old_idhentry = idhentry; - while ((idhentry = (PartRelCacheEntry*)hash_seq_search(&status)) != NULL) { - if (old_idhentry == idhentry) { - break; - } - } - } - } } } @@ -1208,7 +1156,6 @@ static void PartRelCacheInitialize(void) u_sess->cache_cxt.PartRelCache = hash_create("PartRelcache by OID", INITPARTRELCACHESIZE, &ctl, HASH_ELEM | HASH_FUNCTION | HASH_CONTEXT); CacheRegisterSessionPartcacheCallback(PartRelCachePdIdCallback, (Datum)0); - CacheRegisterSessionRelcacheCallback(PartRelCacheRdIdCallback, (Datum)0); } } @@ -1265,6 +1212,7 @@ void partitionInitPartRel(Relation rel, Partition part) idhentry->reldesc->come_from_partrel = true; part->partrel = idhentry->reldesc; part->partrel->rd_refcnt++; + part->partrel->rd_att->tdrefcount++; return; } diff --git a/src/common/backend/utils/cache/relcache.cpp b/src/common/backend/utils/cache/relcache.cpp index 1db7b073a..df44246d9 100755 --- a/src/common/backend/utils/cache/relcache.cpp +++ b/src/common/backend/utils/cache/relcache.cpp @@ -4055,8 +4055,6 @@ void RelationCacheInvalidate(void) /* Delete this entry immediately */ Assert(!relation->rd_isnailed); RelationClearRelation(relation, false); - hash_seq_term(&status); - hash_seq_init(&status, u_sess->relcache_cxt.RelationIdCache); } else { /* * If it's a mapped relation, immediately update its rd_node in @@ -4151,19 +4149,7 @@ void InvalidateRelationNodeList() relation = idhentry->reldesc; if (relation->rd_locator_info != NULL) { - bool clear = RelationHasReferenceCountZero(relation) && - !(RelationIsIndex(relation) && relation->rd_refcnt > 0 && relation->rd_indexcxt != NULL); RelationClearRelation(relation, !RelationHasReferenceCountZero(relation)); - if (!clear) { - hash_seq_term(&status); - hash_seq_init(&status, u_sess->relcache_cxt.RelationIdCache); - while ((idhentry = (RelIdCacheEnt*)hash_seq_search(&status)) != NULL) { - Relation start_relation = idhentry->reldesc; - if (start_relation == relation) { - break; - } - } - } } } } @@ -4394,8 +4380,6 @@ void AtEOXact_RelationCache(bool isCommit) relation->rd_createSubid = InvalidSubTransactionId; } else if (RelationHasReferenceCountZero(relation)) { RelationClearRelation(relation, false); - hash_seq_term(&status); - hash_seq_init(&status, u_sess->relcache_cxt.RelationIdCache); continue; } else { /* @@ -4430,8 +4414,6 @@ void AtEOXact_RelationCache(bool isCommit) } if (relation->partMap != NULL && relation->partMap->isDirty) { RelationClearRelation(relation, false); - hash_seq_term(&status); - hash_seq_init(&status, u_sess->relcache_cxt.RelationIdCache); } } diff --git a/src/gausskernel/runtime/opfusion/opfusion_indexonlyscan.cpp b/src/gausskernel/runtime/opfusion/opfusion_indexonlyscan.cpp index 004b9afa8..6bace98ab 100644 --- a/src/gausskernel/runtime/opfusion/opfusion_indexonlyscan.cpp +++ b/src/gausskernel/runtime/opfusion/opfusion_indexonlyscan.cpp @@ -331,6 +331,8 @@ void IndexOnlyScanFusion::End(bool isCompleted) partitionClose(m_parentIndex, m_partIndex, NoLock); if (!PARTITION_ENABLE_CACHE_OPFUSION) { releaseDummyRelation(&m_index); + } else { + m_index->rd_refcnt--; } index_close(m_parentIndex, NoLock); } else { @@ -343,6 +345,8 @@ void IndexOnlyScanFusion::End(bool isCompleted) partitionClose(m_parentRel, m_partRel, NoLock); if (!PARTITION_ENABLE_CACHE_OPFUSION) { releaseDummyRelation(&m_rel); + } else { + m_rel->rd_refcnt--; } heap_close(m_parentRel, NoLock); } else { diff --git a/src/gausskernel/runtime/opfusion/opfusion_indexscan.cpp b/src/gausskernel/runtime/opfusion/opfusion_indexscan.cpp index 5120073db..ee9117dba 100644 --- a/src/gausskernel/runtime/opfusion/opfusion_indexscan.cpp +++ b/src/gausskernel/runtime/opfusion/opfusion_indexscan.cpp @@ -272,6 +272,8 @@ void IndexScanFusion::End(bool isCompleted) partitionClose(m_parentIndex, m_partIndex, NoLock); if (!PARTITION_ENABLE_CACHE_OPFUSION) { releaseDummyRelation(&m_index); + } else { + m_index->rd_refcnt--; } index_close(m_parentIndex, NoLock); } else { @@ -284,6 +286,8 @@ void IndexScanFusion::End(bool isCompleted) partitionClose(m_parentRel, m_partRel, NoLock); if (!PARTITION_ENABLE_CACHE_OPFUSION) { releaseDummyRelation(&m_rel); + } else { + m_rel->rd_refcnt--; } heap_close(m_parentRel, NoLock); } else { @@ -336,4 +340,4 @@ void IndexScanFusion::ResetIndexScanFusion(IndexScan* node, PlannedStmt* planstm setAttrNo(); m_can_reused = true; -} \ No newline at end of file +} diff --git a/src/gausskernel/runtime/opfusion/opfusion_util.cpp b/src/gausskernel/runtime/opfusion/opfusion_util.cpp index 4b4dee292..98f3568fc 100644 --- a/src/gausskernel/runtime/opfusion/opfusion_util.cpp +++ b/src/gausskernel/runtime/opfusion/opfusion_util.cpp @@ -1348,6 +1348,7 @@ Relation InitPartitionIndexInFusion(Oid parentIndexOid, Oid partOid, Partition * } (*partIndex)->partrel->pgstat_info = (*partIndex)->pd_pgstat_info; index = (*partIndex)->partrel; + index->rd_refcnt++; } else { index = partitionGetRelation(*parentIndex, *partIndex); } @@ -1366,6 +1367,7 @@ void InitPartitionRelationInFusion(Oid partOid, Relation parentRel, Partition* p } (*partRel)->partrel->pgstat_info = (*partRel)->pd_pgstat_info; *rel = (*partRel)->partrel; + (*rel)->rd_refcnt++; } else { *rel = partitionGetRelation(parentRel, *partRel); } @@ -1379,11 +1381,15 @@ void ExeceDoneInIndexFusionConstruct(bool isPartTbl, Relation* parentRel, Partit if (*index != NULL) { if (!PARTITION_ENABLE_CACHE_OPFUSION) { releaseDummyRelation(index); + } else { + (*index)->rd_refcnt--; } *index = NULL; } if (!PARTITION_ENABLE_CACHE_OPFUSION) { releaseDummyRelation(rel); + } else { + (*rel)->rd_refcnt--; } partitionClose(*parentRel, *part, AccessShareLock); heap_close(*parentRel, AccessShareLock); From 0cff35d191f5b5991d8a26a8961567812f2bf695 Mon Sep 17 00:00:00 2001 From: gbzhangkai Date: Tue, 11 Jul 2023 13:36:56 +0800 Subject: [PATCH 017/304] =?UTF-8?q?=E4=BD=BF=E8=83=BD=E7=BC=96=E8=AF=91?= =?UTF-8?q?=E9=80=89=E9=A1=B9FDDEBUG=E5=90=8E=EF=BC=8C=E7=BC=96=E8=AF=91?= =?UTF-8?q?=E4=B8=8D=E8=BF=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/storage/file/fd.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gausskernel/storage/file/fd.cpp b/src/gausskernel/storage/file/fd.cpp index 7864942f4..163a1be4f 100755 --- a/src/gausskernel/storage/file/fd.cpp +++ b/src/gausskernel/storage/file/fd.cpp @@ -2756,7 +2756,7 @@ int AllocateSocket(const char* ipaddr, int port) errno_t rc = EOK; int retrynum = 0; - DO_DB(ereport(LOG, (errmsg("AllocateFile: Allocated %d (%s)", u_sess->storage_cxt.numAllocatedDescs, name)))); + DO_DB(ereport(LOG, (errmsg("AllocateFile: Allocated %d", u_sess->storage_cxt.numAllocatedDescs)))); Assert(ipaddr != NULL); restart: From be0cc82f51ea98ab76ed9534653e27e4194b3dd7 Mon Sep 17 00:00:00 2001 From: ytwx1993 Date: Tue, 11 Jul 2023 13:53:28 +0800 Subject: [PATCH 018/304] =?UTF-8?q?intervaltypmodin=E5=87=BD=E6=95=B0?= =?UTF-8?q?=E4=B8=AD=E4=BD=BF=E7=94=A8unsigned=20int32=E7=B1=BB=E5=9E=8B?= =?UTF-8?q?=E5=AD=98=E5=9C=A8=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/utils/adt/timestamp.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/common/backend/utils/adt/timestamp.cpp b/src/common/backend/utils/adt/timestamp.cpp index 673db2490..168ba556e 100644 --- a/src/common/backend/utils/adt/timestamp.cpp +++ b/src/common/backend/utils/adt/timestamp.cpp @@ -1111,7 +1111,7 @@ Datum intervaltypmodin(PG_FUNCTION_ARGS) if (n == 1) { if (tl[0] != INTERVAL_FULL_RANGE) - typmod = INTERVAL_TYPMOD(INTERVAL_FULL_PRECISION, (unsigned int32)tl[0]); + typmod = INTERVAL_TYPMOD(INTERVAL_FULL_PRECISION, (unsigned int)tl[0]); else typmod = -1; } else if (n == 2) { @@ -1123,9 +1123,9 @@ Datum intervaltypmodin(PG_FUNCTION_ARGS) ereport(WARNING, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("INTERVAL(%d) precision reduced to maximum allowed, %d", tl[1], MAX_INTERVAL_PRECISION))); - typmod = INTERVAL_TYPMOD(MAX_INTERVAL_PRECISION, (unsigned int32)tl[0]); + typmod = INTERVAL_TYPMOD(MAX_INTERVAL_PRECISION, (unsigned int)tl[0]); } else - typmod = INTERVAL_TYPMOD((unsigned int32)tl[1], (unsigned int32)tl[0]); + typmod = INTERVAL_TYPMOD((unsigned int)tl[1], (unsigned int)tl[0]); } else { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid INTERVAL type modifier"))); typmod = 0; /* keep compiler quiet */ From 2aa100780ff7eda80161f102f3aae977639445e1 Mon Sep 17 00:00:00 2001 From: zhang_xubo <2578876417@qq.com> Date: Tue, 11 Jul 2023 14:52:21 +0800 Subject: [PATCH 019/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E4=BF=AE=E6=94=B9rep?= =?UTF-8?q?l=E4=B8=BA=E7=A9=BA=E6=97=B6=E5=81=9C=E6=AD=A2=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E5=BA=93=E4=BA=A7=E7=94=9Fcore=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/utils/misc/guc/guc_storage.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/common/backend/utils/misc/guc/guc_storage.cpp b/src/common/backend/utils/misc/guc/guc_storage.cpp index 196b32fb8..0826dc7de 100755 --- a/src/common/backend/utils/misc/guc/guc_storage.cpp +++ b/src/common/backend/utils/misc/guc/guc_storage.cpp @@ -5007,8 +5007,6 @@ static int IsReplConnInfoChanged(const char* replConnInfo, const char* newval) int repl_length = 0; replconninfo* newReplInfo = NULL; replconninfo* ReplInfo_1 = t_thrd.postmaster_cxt.ReplConnArray[1]; - newval = TrimStr(newval); - replConnInfo = TrimStr(replConnInfo); if (replConnInfo == NULL || newval == NULL) { return NO_CHANGE; } @@ -5028,9 +5026,9 @@ static int IsReplConnInfoChanged(const char* replConnInfo, const char* newval) return ADD_REPL_CONN_INFO_WITH_NEW_LOCAL_IP_PORT; } - if (strcmp(ReplInfo_1->localhost, newReplInfo->localhost) != 0 || + if (newReplInfo != NULL && (strcmp(ReplInfo_1->localhost, newReplInfo->localhost) != 0 || ReplInfo_1->localport != newReplInfo->localport || - ReplInfo_1->localheartbeatport != newReplInfo->localheartbeatport) { + ReplInfo_1->localheartbeatport != newReplInfo->localheartbeatport)) { pfree_ext(newReplInfo); pfree_ext(oldReplStr); pfree_ext(newReplStr); From e99f2b8aece3869d9656ebf8755da56a4ccd3871 Mon Sep 17 00:00:00 2001 From: yyl164119487 Date: Wed, 12 Jul 2023 14:38:47 +0800 Subject: [PATCH 020/304] =?UTF-8?q?=E7=83=AD=E7=82=B9=E5=87=BD=E6=95=B0?= =?UTF-8?q?=E6=9B=BF=E6=8D=A2memcpy=5Fsp=E5=92=8C=E6=B7=BB=E5=8A=A0inline?= =?UTF-8?q?=E5=87=BD=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/lib/stringinfo.cpp | 29 ------------------- src/common/backend/libpq/pqformat.cpp | 17 ++--------- src/common/backend/utils/adt/numutils.cpp | 10 +++---- .../runtime/executor/execTuples.cpp | 25 ---------------- .../runtime/executor/nodeIndexonlyscan.cpp | 4 +-- .../storage/access/heap/heapam.cpp | 10 +++---- .../storage/access/nbtree/nbtsearch.cpp | 4 +-- src/include/executor/tuptable.h | 28 +++++++++++++++++- src/include/gtm/utils/stringinfo.h | 27 +++++++++++++++-- src/include/lib/stringinfo.h | 21 ++++++++++++-- src/include/libpq/pqformat.h | 18 ++++++++++-- 11 files changed, 101 insertions(+), 92 deletions(-) diff --git a/src/common/backend/lib/stringinfo.cpp b/src/common/backend/lib/stringinfo.cpp index 3fb8bb771..cfe298040 100644 --- a/src/common/backend/lib/stringinfo.cpp +++ b/src/common/backend/lib/stringinfo.cpp @@ -68,19 +68,6 @@ void FreeStringInfo(StringInfo str) } } -/* - * resetStringInfo - * - * Reset the StringInfo: the data buffer remains valid, but its - * previous content, if any, is cleared. - */ -void resetStringInfo(StringInfo str) -{ - str->data[0] = '\0'; - str->len = 0; - str->cursor = 0; -} - /* * appendStringInfo * @@ -550,22 +537,6 @@ void enlargeBufferSize(int needed, // needed more bytes // template void enlargeBuffer(int, int, int*, char** data); // template void enlargeBuffer(int, int, size_t*, char** data); -/* - * enlargeStringInfo - * - * Make sure there is enough space for StringInfo - * - * External callers usually need not concern themselves with this, since - * all stringinfo.c routines do it automatically. However, if a caller - * knows that a StringInfo will eventually become X bytes large, it - * can save some palloc overhead by enlarging the buffer before starting - * to store data in it. - */ -void enlargeStringInfo(StringInfo str, int needed) -{ - enlargeBuffer(needed, str->len, &str->maxlen, &str->data); -} - /* * The following function is used to request the use of more than 1GB of memory */ diff --git a/src/common/backend/libpq/pqformat.cpp b/src/common/backend/libpq/pqformat.cpp index 64ad3678e..eaaa9dbfa 100644 --- a/src/common/backend/libpq/pqformat.cpp +++ b/src/common/backend/libpq/pqformat.cpp @@ -163,7 +163,7 @@ void pq_sendcountedtext_printtup(StringInfo buf, const char* str, int slen, int slen = strlen(p); enlargeStringInfo(buf, slen + sizeof(uint32)); pq_writeint32(buf, (uint32)slen); - errno_t rc = memcpy_s(buf->data + buf->len, (size_t)(buf->maxlen - buf->len), p, (size_t)slen); + errno_t rc = memcpy_sp(buf->data + buf->len, (size_t)(buf->maxlen - buf->len), p, (size_t)slen); securec_check(rc, "\0", "\0"); buf->len += slen; buf->data[buf->len] = '\0'; @@ -172,7 +172,7 @@ void pq_sendcountedtext_printtup(StringInfo buf, const char* str, int slen, int } else { enlargeStringInfo(buf, slen + sizeof(uint32)); pq_writeint32(buf, (uint32)slen); - errno_t rc = memcpy_s(buf->data + buf->len, (size_t)(buf->maxlen - buf->len), str, (size_t)slen); + errno_t rc = memcpy_sp(buf->data + buf->len, (size_t)(buf->maxlen - buf->len), str, (size_t)slen); securec_check(rc, "\0", "\0"); buf->len += slen; buf->data[buf->len] = '\0'; @@ -317,19 +317,6 @@ void pq_endmessage(StringInfo buf) buf->data = NULL; } -/* -------------------------------- - * pq_endmessage_reuse - send the completed message to the frontend - * - * The data buffer is *not* freed, allowing to reuse the buffer with - * pg_beginmessage_reuse. - -------------------------------- - */ -void pq_endmessage_reuse(StringInfo buf) -{ - /* msgtype was saved in cursor field */ - (void)pq_putmessage(buf->cursor, buf->data, buf->len); -} - /* -------------------------------- * pq_endmessage_noblock - Same as pq_endmessage but non-blocking * -------------------------------- diff --git a/src/common/backend/utils/adt/numutils.cpp b/src/common/backend/utils/adt/numutils.cpp index 4b7ed34ee..fc581008a 100644 --- a/src/common/backend/utils/adt/numutils.cpp +++ b/src/common/backend/utils/adt/numutils.cpp @@ -545,9 +545,9 @@ int pg_ultoa_n(uint32 value, char *a) value /= 10000; - errno_t rc = memcpy_s(pos - 2, 2, DIGIT_TABLE + c0, 2); + errno_t rc = memcpy_sp(pos - 2, 2, DIGIT_TABLE + c0, 2); securec_check(rc, "\0", "\0"); - rc = memcpy_s(pos - 4, 2, DIGIT_TABLE + c1, 2); + rc = memcpy_sp(pos - 4, 2, DIGIT_TABLE + c1, 2); securec_check(rc, "\0", "\0"); i += 4; } @@ -558,7 +558,7 @@ int pg_ultoa_n(uint32 value, char *a) value /= 100; - errno_t rc = memcpy_s(pos - 2, 2, DIGIT_TABLE + c, 2); + errno_t rc = memcpy_sp(pos - 2, 2, DIGIT_TABLE + c, 2); securec_check(rc, "\0", "\0"); i += 2; } @@ -567,7 +567,7 @@ int pg_ultoa_n(uint32 value, char *a) char *pos = a + olength - i; - errno_t rc = memcpy_s(pos - 2, 2, DIGIT_TABLE + c, 2); + errno_t rc = memcpy_sp(pos - 2, 2, DIGIT_TABLE + c, 2); securec_check(rc, "\0", "\0"); } else { *a = (char)('0' + value); @@ -633,7 +633,7 @@ char* pg_ultostr_zeropad(char* str, uint32 value, int32 minwidth) if (value < 100 && minwidth == 2) /* Short cut for common case */ { - rc = memcpy_s(str, 2, DIGIT_TABLE + value * 2, 2); + rc = memcpy_sp(str, 2, DIGIT_TABLE + value * 2, 2); securec_check(rc, "\0", "\0"); return str + 2; } diff --git a/src/gausskernel/runtime/executor/execTuples.cpp b/src/gausskernel/runtime/executor/execTuples.cpp index 741c8e516..c55f82625 100644 --- a/src/gausskernel/runtime/executor/execTuples.cpp +++ b/src/gausskernel/runtime/executor/execTuples.cpp @@ -483,31 +483,6 @@ void ExecClearMutilTuple(List* slots) (void)ExecClearTuple(slot); } } -/* -------------------------------- - * ExecStoreVirtualTuple - * Mark a slot as containing a virtual tuple. - * - * The protocol for loading a slot with virtual tuple data is: - * * Call ExecClearTuple to mark the slot empty. - * * Store data into the Datum/isnull arrays. - * * Call ExecStoreVirtualTuple to mark the slot valid. - * This is a bit unclean but it avoids one round of data copying. - * -------------------------------- - */ -TupleTableSlot* ExecStoreVirtualTuple(TupleTableSlot* slot) -{ - /* - * sanity checks - */ - Assert(slot != NULL); - Assert(slot->tts_tupleDescriptor != NULL); - Assert(TTS_EMPTY(slot)); - - slot->tts_flags &= ~TTS_FLAG_EMPTY; - slot->tts_nvalid = slot->tts_tupleDescriptor->natts; - - return slot; -} /* -------------------------------- * ExecStoreAllNullTuple diff --git a/src/gausskernel/runtime/executor/nodeIndexonlyscan.cpp b/src/gausskernel/runtime/executor/nodeIndexonlyscan.cpp index 75fd68b7e..a15e60519 100644 --- a/src/gausskernel/runtime/executor/nodeIndexonlyscan.cpp +++ b/src/gausskernel/runtime/executor/nodeIndexonlyscan.cpp @@ -66,7 +66,7 @@ static inline void ReleaseNodeVMBuffer(IndexOnlyScanState* node) * ExecGPIGetNextPartRelation * ---------------------------------------------------------------- */ -bool ExecGPIGetNextPartRelation(IndexOnlyScanState* node, IndexScanDesc indexScan) +inline bool ExecGPIGetNextPartRelation(IndexOnlyScanState* node, IndexScanDesc indexScan) { if (IndexScanNeedSwitchPartRel(indexScan)) { /* Release VM buffer pin, if any. */ @@ -334,7 +334,7 @@ static bool IndexOnlyRecheck(IndexOnlyScanState* node, TupleTableSlot* slot) * ExecIndexOnlyScan(node) * ---------------------------------------------------------------- */ -static TupleTableSlot* ExecIndexOnlyScan(PlanState* state) +static inline TupleTableSlot* ExecIndexOnlyScan(PlanState* state) { IndexOnlyScanState* node = castNode(IndexOnlyScanState, state); /* diff --git a/src/gausskernel/storage/access/heap/heapam.cpp b/src/gausskernel/storage/access/heap/heapam.cpp index e2bda60e3..cda1455f3 100755 --- a/src/gausskernel/storage/access/heap/heapam.cpp +++ b/src/gausskernel/storage/access/heap/heapam.cpp @@ -2431,7 +2431,7 @@ bool heap_hot_search_buffer(ItemPointer tid, Relation relation, Buffer buffer, S /* check for bogus TID */ if (offnum < FirstOffsetNumber || offnum > PageGetMaxOffsetNumber(dp)) { - break; + return false; } lp = PageGetItemId(dp, offnum); @@ -2445,7 +2445,7 @@ bool heap_hot_search_buffer(ItemPointer tid, Relation relation, Buffer buffer, S continue; } /* else must be end of chain */ - break; + return false; } /* @@ -2468,7 +2468,7 @@ bool heap_hot_search_buffer(ItemPointer tid, Relation relation, Buffer buffer, S * Shouldn't see a HEAP_ONLY tuple at chain start. */ if (at_chain_start && HeapTupleIsHeapOnly(heap_tuple)) { - break; + return false; } /* @@ -2476,7 +2476,7 @@ bool heap_hot_search_buffer(ItemPointer tid, Relation relation, Buffer buffer, S * broken. */ if (TransactionIdIsValid(prev_xmax) && !TransactionIdEquals(prev_xmax, HeapTupleGetRawXmin(heap_tuple))) { - break; + return false; } /* @@ -2557,7 +2557,7 @@ bool heap_hot_search_buffer(ItemPointer tid, Relation relation, Buffer buffer, S at_chain_start = false; prev_xmax = HeapTupleGetUpdateXid(heap_tuple); } else { - break; /* end of chain */ + return false; /* end of chain */ } } return false; diff --git a/src/gausskernel/storage/access/nbtree/nbtsearch.cpp b/src/gausskernel/storage/access/nbtree/nbtsearch.cpp index de4376f5e..14a52d5dd 100644 --- a/src/gausskernel/storage/access/nbtree/nbtsearch.cpp +++ b/src/gausskernel/storage/access/nbtree/nbtsearch.cpp @@ -1852,7 +1852,7 @@ bool _bt_check_natts(const Relation index, bool heapkeyspace, Page page, OffsetN return num_tuple_attrs > 0 && num_tuple_attrs <= nkeyatts; } -static int btree_setup_posting_items(BTScanOpaque so, int item_idx, OffsetNumber offnum, ItemPointer heap_tid, +static inline int btree_setup_posting_items(BTScanOpaque so, int item_idx, OffsetNumber offnum, ItemPointer heap_tid, IndexTuple tuple) { BTScanPosItem *curr_item = &so->currPos.items[item_idx]; @@ -1878,7 +1878,7 @@ static int btree_setup_posting_items(BTScanOpaque so, int item_idx, OffsetNumber return 0; } -static void btree_save_posting_item(BTScanOpaque so, int item_idx, OffsetNumber offnum, ItemPointer heap_tid, +static inline void btree_save_posting_item(BTScanOpaque so, int item_idx, OffsetNumber offnum, ItemPointer heap_tid, int tuple_offset) { BTScanPosItem *curr_item = &so->currPos.items[item_idx]; diff --git a/src/include/executor/tuptable.h b/src/include/executor/tuptable.h index 6dec954e0..8b83e9a20 100644 --- a/src/include/executor/tuptable.h +++ b/src/include/executor/tuptable.h @@ -196,7 +196,33 @@ extern TupleTableSlot* ExecStoreDataRowTuple( #endif extern TupleTableSlot* ExecClearTuple(TupleTableSlot* slot); extern void ExecClearMutilTuple(List* slots); -extern TupleTableSlot* ExecStoreVirtualTuple(TupleTableSlot* slot); + +/* -------------------------------- + * ExecStoreVirtualTuple + * Mark a slot as containing a virtual tuple. + * + * The protocol for loading a slot with virtual tuple data is: + * * Call ExecClearTuple to mark the slot empty. + * * Store data into the Datum/isnull arrays. + * * Call ExecStoreVirtualTuple to mark the slot valid. + * This is a bit unclean but it avoids one round of data copying. + * -------------------------------- + */ +inline TupleTableSlot* ExecStoreVirtualTuple(TupleTableSlot* slot) +{ + /* + * sanity checks + */ + Assert(slot != NULL); + Assert(slot->tts_tupleDescriptor != NULL); + Assert(TTS_EMPTY(slot)); + + slot->tts_flags &= ~TTS_FLAG_EMPTY; + slot->tts_nvalid = slot->tts_tupleDescriptor->natts; + + return slot; +} + extern TupleTableSlot* ExecStoreAllNullTuple(TupleTableSlot* slot); extern HeapTuple ExecCopySlotTuple(TupleTableSlot* slot); extern MinimalTuple ExecCopySlotMinimalTuple(TupleTableSlot* slot, bool need_transform_anyarray = false); diff --git a/src/include/gtm/utils/stringinfo.h b/src/include/gtm/utils/stringinfo.h index 394206078..48c978f43 100644 --- a/src/include/gtm/utils/stringinfo.h +++ b/src/include/gtm/utils/stringinfo.h @@ -93,7 +93,12 @@ extern void initStringInfo(StringInfo str); * Clears the current content of the StringInfo, if any. The * StringInfo remains valid. */ -extern void resetStringInfo(StringInfo str); +inline void resetStringInfo(StringInfo str) +{ + str->data[0] = '\0'; + str->len = 0; + str->cursor = 0; +} /* ------------------------ * appendStringInfo @@ -146,11 +151,27 @@ extern void appendStringInfoChar(StringInfo str, char ch); */ extern void appendBinaryStringInfo(StringInfo str, const char* data, int datalen); +/* ------------------------ + * enlargeBuffer + * Make sure a buffer can hold at least 'needed' more bytes. + */ +void enlargeBuffer(int needed, int len, int* maxlen, char** data); + /* ------------------------ * enlargeStringInfo - * Make sure a StringInfo's buffer can hold at least 'needed' more bytes. + * + * Make sure there is enough space for StringInfo + * + * External callers usually need not concern themselves with this, since + * all stringinfo.c routines do it automatically. However, if a caller + * knows that a StringInfo will eventually become X bytes large, it + * can save some palloc overhead by enlarging the buffer before starting + * to store data in it. */ -extern void enlargeStringInfo(StringInfo str, int needed); +inline void enlargeStringInfo(StringInfo str, int needed) +{ + enlargeBuffer(needed, str->len, &str->maxlen, &str->data); +} /* ----------------------- * dupStringInfo diff --git a/src/include/lib/stringinfo.h b/src/include/lib/stringinfo.h index b26f6400e..313aff68e 100644 --- a/src/include/lib/stringinfo.h +++ b/src/include/lib/stringinfo.h @@ -87,7 +87,12 @@ extern void FreeStringInfo(StringInfo str); * Clears the current content of the StringInfo, if any. The * StringInfo remains valid. */ -extern void resetStringInfo(StringInfo str); +inline void resetStringInfo(StringInfo str) +{ + str->data[0] = '\0'; + str->len = 0; + str->cursor = 0; +} /* ------------------------ * appendStringInfo @@ -166,9 +171,19 @@ void enlargeBufferSize(int needed, int len, size_t* maxlen, char** data); /* ------------------------ * enlargeStringInfo - * Make sure a StringInfo's buffer can hold at least 'needed' more bytes. + * + * Make sure there is enough space for StringInfo + * + * External callers usually need not concern themselves with this, since + * all stringinfo.c routines do it automatically. However, if a caller + * knows that a StringInfo will eventually become X bytes large, it + * can save some palloc overhead by enlarging the buffer before starting + * to store data in it. */ -extern void enlargeStringInfo(StringInfo str, int needed); +inline void enlargeStringInfo(StringInfo str, int needed) +{ + enlargeBuffer(needed, str->len, &str->maxlen, &str->data); +} /* * The following function is used to request the use of more than 1GB of memory diff --git a/src/include/libpq/pqformat.h b/src/include/libpq/pqformat.h index 287c065ef..178fd62fe 100644 --- a/src/include/libpq/pqformat.h +++ b/src/include/libpq/pqformat.h @@ -28,8 +28,9 @@ extern void pq_send_ascii_string(StringInfo buf, const char* str); extern void pq_sendfloat4(StringInfo buf, float4 f); extern void pq_sendfloat8(StringInfo buf, float8 f); extern void pq_endmessage(StringInfo buf); -extern void pq_endmessage_reuse(StringInfo buf); extern void pq_endmessage_noblock(StringInfo buf); +extern int pq_putmessage(char msgtype, const char* s, size_t len); + extern void pq_begintypsend(StringInfo buf); extern bytea* pq_endtypsend(StringInfo buf); @@ -51,6 +52,19 @@ extern char* pq_getmsgtext(StringInfo msg, int rawbytes, int* nbytes); extern const char* pq_getmsgstring(StringInfo msg); extern void pq_getmsgend(StringInfo msg); +/* -------------------------------- + * pq_endmessage_reuse - send the completed message to the frontend + * + * The data buffer is *not* freed, allowing to reuse the buffer with + * pg_beginmessage_reuse. + -------------------------------- + */ +inline void pq_endmessage_reuse(StringInfo buf) +{ + /* msgtype was saved in cursor field */ + (void)pq_putmessage(buf->cursor, buf->data, buf->len); +} + static inline uint128 pg_bswap128(uint128 x) { return @@ -125,7 +139,7 @@ static inline void pq_writeint32(StringInfoData* pg_restrict buf, uint32 i) uint32 ni = htonl(i); errno_t rc = EOK; - rc = memcpy_s((char* pg_restrict)(buf->data + buf->len), sizeof(uint32), &ni, sizeof(uint32)); + rc = memcpy_sp((char* pg_restrict)(buf->data + buf->len), sizeof(uint32), &ni, sizeof(uint32)); securec_check(rc, "\0", "\0"); buf->len += sizeof(uint32); } From a08847a7b58070f3a3de76c9c192359d52dfed3b Mon Sep 17 00:00:00 2001 From: WangXiuqiang Date: Thu, 13 Jul 2023 15:41:39 +0800 Subject: [PATCH 021/304] =?UTF-8?q?=E8=A7=A3=E5=86=B3datcompatibility=3D'B?= =?UTF-8?q?'=20=E5=88=9B=E5=BB=BA=E5=88=86=E5=8C=BA=E8=A1=A8=EF=BC=8C?= =?UTF-8?q?=E4=BD=BF=E7=94=A8gs=5Fdump=E5=AF=BC=E5=87=BA=E6=8A=A5=E9=94=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/pg_dump/pg_dump.cpp | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/bin/pg_dump/pg_dump.cpp b/src/bin/pg_dump/pg_dump.cpp index ad1254bf7..87616b0af 100644 --- a/src/bin/pg_dump/pg_dump.cpp +++ b/src/bin/pg_dump/pg_dump.cpp @@ -18338,12 +18338,19 @@ static PQExpBuffer createTablePartition(Archive* fout, TableInfo* tbinfo) PART_OBJ_TYPE_TABLE_PARTITION, newStrategy); for (i = 1; i <= partkeynum; i++) { - if (i == partkeynum) - appendPQExpBuffer( - partitionq, "p.boundaries[%d]::%s ASC", i, tbinfo->atttypnames[partkeycols[i - 1] - 1]); - else - appendPQExpBuffer( - partitionq, "p.boundaries[%d]::%s, ", i, tbinfo->atttypnames[partkeycols[i - 1] - 1]); + if (partkeyexprIsNull) { + if (i == partkeynum) + appendPQExpBuffer( + partitionq, "p.boundaries[%d]::%s ASC", i, tbinfo->atttypnames[partkeycols[i - 1] - 1]); + else + appendPQExpBuffer( + partitionq, "p.boundaries[%d]::%s, ", i, tbinfo->atttypnames[partkeycols[i - 1] - 1]); + } else { + if (i == partkeynum) + appendPQExpBuffer(partitionq, "p.boundaries[%d] ASC", i); + else + appendPQExpBuffer(partitionq, "p.boundaries[%d], ", i); + } } } else { appendPQExpBuffer(partitionq, From 09288b9ff8aa4cdcbd47e4046aad2a66d5a419a9 Mon Sep 17 00:00:00 2001 From: gbzhangkai Date: Thu, 13 Jul 2023 19:47:32 +0800 Subject: [PATCH 022/304] =?UTF-8?q?=E5=87=BD=E6=95=B0FlushBuffer=E4=B8=AD?= =?UTF-8?q?=E7=BB=93=E6=9E=84=E4=BD=93=E5=8F=98=E9=87=8F=E5=88=9D=E5=A7=8B?= =?UTF-8?q?=E5=8C=96=E4=B8=8D=E6=AD=A3=E7=A1=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/storage/buffer/bufmgr.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/gausskernel/storage/buffer/bufmgr.cpp b/src/gausskernel/storage/buffer/bufmgr.cpp index a4a8b2242..311a80eb8 100644 --- a/src/gausskernel/storage/buffer/bufmgr.cpp +++ b/src/gausskernel/storage/buffer/bufmgr.cpp @@ -4666,7 +4666,9 @@ void FlushBuffer(void *buf, SMgrRelation reln, ReadBufferMethod flushmethod, boo Block bufBlock; char *bufToWrite = NULL; uint32 buf_state; - RedoBufferInfo bufferinfo = {0}; + RedoBufferInfo bufferinfo; + errno_t rc = memset_s(&bufferinfo, sizeof(RedoBufferInfo), 0, sizeof(RedoBufferInfo)); + securec_check(rc, "\0", "\0"); t_thrd.dms_cxt.buf_in_aio = false; From 4af20e14db5fe2d712d172e3c9895744f2a4a23e Mon Sep 17 00:00:00 2001 From: ytwx1993 Date: Fri, 14 Jul 2023 15:17:05 +0800 Subject: [PATCH 023/304] =?UTF-8?q?=E8=8E=B7=E5=8F=96CPU=E4=BF=A1=E6=81=AF?= =?UTF-8?q?=E8=8B=A5env=E4=B8=AD=E5=8C=85=E5=90=ABLANGUAGE=E9=9C=80?= =?UTF-8?q?=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/process/threadpool/threadpool_controler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gausskernel/process/threadpool/threadpool_controler.cpp b/src/gausskernel/process/threadpool/threadpool_controler.cpp index 687be8ab9..0d3d29107 100644 --- a/src/gausskernel/process/threadpool/threadpool_controler.cpp +++ b/src/gausskernel/process/threadpool/threadpool_controler.cpp @@ -575,7 +575,7 @@ void ThreadPoolControler::GetCpuAndNumaNum(int32 *totalCpuNum, int32 *totalNumaN FILE* fp = NULL; - if ((fp = popen("LANG=en_US.UTF-8;lscpu", "r")) != NULL) { + if ((fp = popen("LANGUAGE=en_US.UTF-8;LANG=en_US.UTF-8;lscpu", "r")) != NULL) { while (fgets(buf, sizeof(buf), fp) != NULL) { if (strncmp("CPU(s)", buf, strlen("CPU(s)")) == 0 && strncmp("On-line CPU(s) list", buf, strlen("On-line CPU(s) list")) != 0 && From 501bc43cc5a27470520ab2950865f4d060fa075e Mon Sep 17 00:00:00 2001 From: gentle_hu Date: Mon, 17 Jul 2023 20:30:54 +0800 Subject: [PATCH 024/304] Fix the issue of memory overlap when copying cgroup strings --- src/bin/gs_cgroup/cgexec.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/bin/gs_cgroup/cgexec.cpp b/src/bin/gs_cgroup/cgexec.cpp index ae85209f9..9dde56179 100644 --- a/src/bin/gs_cgroup/cgexec.cpp +++ b/src/bin/gs_cgroup/cgexec.cpp @@ -1054,9 +1054,11 @@ static int cgexec_create_remain_cgroup(gscgroup_grp_t* grp) cpushares = MAX_CLASS_CPUSHARES * cgutil_vaddr[i]->ginfo.cls.rempct / GROUP_ALL_PERCENT; ioweight = IO_WEIGHT_CALC(MAX_IO_WEIGHT, cgutil_vaddr[i]->ginfo.cls.rempct); - sret = - snprintf_s(cgutil_vaddr[i]->cpuset, CPUSET_LEN, CPUSET_LEN - 1, "%s", cgutil_vaddr[grp->ginfo.wd.cgid]->cpuset); - securec_check_intval(sret, free(relpath), -1); + if (i != grp->ginfo.wd.cgid) { + sret = snprintf_s(cgutil_vaddr[i]->cpuset, CPUSET_LEN, CPUSET_LEN - 1, + "%s", cgutil_vaddr[grp->ginfo.wd.cgid]->cpuset); + securec_check_intval(sret, free(relpath), -1); + } (void)cgexec_create_default_cgroup(relpath, cpushares, ioweight, cgutil_vaddr[i]->cpuset); From 02d5adeed938cd9d27f07c05bd0705a7117ca13e Mon Sep 17 00:00:00 2001 From: chenxiaobin19 <1025221611@qq.com> Date: Mon, 17 Jul 2023 21:46:19 +0800 Subject: [PATCH 025/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=9B=B4=E6=96=B0?= =?UTF-8?q?=E8=A7=86=E5=9B=BE=E4=B8=BAdefault=E5=A4=B1=E6=95=88=E7=9A=84?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../optimizer/rewrite/rewriteHandler.cpp | 8 ++++--- src/test/regress/expected/multi_update.out | 23 +++++++++++++++++++ src/test/regress/expected/updatable_views.out | 18 +++++++++++++++ src/test/regress/sql/multi_update.sql | 5 ++++ src/test/regress/sql/updatable_views.sql | 11 +++++++++ 5 files changed, 62 insertions(+), 3 deletions(-) diff --git a/src/gausskernel/optimizer/rewrite/rewriteHandler.cpp b/src/gausskernel/optimizer/rewrite/rewriteHandler.cpp index 4efc0ffe3..60b345ab9 100644 --- a/src/gausskernel/optimizer/rewrite/rewriteHandler.cpp +++ b/src/gausskernel/optimizer/rewrite/rewriteHandler.cpp @@ -791,7 +791,7 @@ static List* rewriteTargetListIU(List* targetList, CmdType commandType, Relation ereport(DEBUG2, (errmodule(MOD_PARSER), errcode(ERRCODE_LOG), errmsg("default column \"%s\" is effectively NULL, and hence omitted.", NameStr(att_tup->attname)))); - } else { + } else if (target_relation->rd_rel->relkind != RELKIND_VIEW) { new_expr = (Node*)makeConst(att_tup->atttypid, -1, att_tup->attcollation, @@ -936,7 +936,7 @@ static void rewriteTargetListMutilUpdate(Query* parsetree, List* rtable, List* r new_tle = NULL; } else if (applyDefault) { Node* new_expr = build_column_default(target_relation, attrno, true); - if (new_expr == NULL) { + if (new_expr == NULL && target_relation->rd_rel->relkind != RELKIND_VIEW) { new_expr = (Node*)makeConst(att_tup->atttypid, -1, att_tup->attcollation, @@ -948,7 +948,9 @@ static void rewriteTargetListMutilUpdate(Query* parsetree, List* rtable, List* r new_expr = coerce_to_domain( new_expr, InvalidOid, -1, att_tup->atttypid, COERCE_IMPLICIT_CAST, -1, false, false); } - new_tle = makeTargetEntry((Expr*)new_expr, attrno, pstrdup(NameStr(att_tup->attname)), false); + if (new_expr != NULL) { + new_tle = makeTargetEntry((Expr*)new_expr, attrno, pstrdup(NameStr(att_tup->attname)), false); + } new_tle->rtindex = result_relation; } diff --git a/src/test/regress/expected/multi_update.out b/src/test/regress/expected/multi_update.out index 848600916..965c0dd0b 100644 --- a/src/test/regress/expected/multi_update.out +++ b/src/test/regress/expected/multi_update.out @@ -1326,6 +1326,29 @@ select * from t_t_mutil_t3; 1 | 6 (2 rows) +alter table t_t_mutil_t1 alter column col2 set default 999; +update t_t_mutil_t1 a,multiview2 b,t_t_mutil_t3 c set a.col2 = default, b.col2 = 5,c.col2 = 6 where a.col1 = b.col1 and a.col1 = c.col1; +select * from t_t_mutil_t1; + col1 | col2 +------+------ + 1 | 999 + 1 | 999 +(2 rows) + +select * from t_t_mutil_t2; + col1 | col2 | col3 +------+------+------ + 1 | 5 | + 1 | 5 | +(2 rows) + +select * from t_t_mutil_t3; + col1 | col2 +------+------ + 1 | 6 + 1 | 6 +(2 rows) + -- left join update multiview1 a left join multiview2 b on a.col1=b.col1 set a.col2=7,b.col2=8; select * from t_t_mutil_t1; diff --git a/src/test/regress/expected/updatable_views.out b/src/test/regress/expected/updatable_views.out index 3404210b7..777d4c9e7 100644 --- a/src/test/regress/expected/updatable_views.out +++ b/src/test/regress/expected/updatable_views.out @@ -2472,5 +2472,23 @@ explain (costs off) UPDATE ro_view1 SET a = a + b; UPDATE ro_view1 SET a = a + b; ERROR: new row violates WITH CHECK OPTION for view "ro_view1" DETAIL: Failing row contains (10, 2). +create table default_t (a int, b varchar default 'table default'); +insert into default_t values (1, 'dec'); +create view default_v as select b, a from default_t; +update default_v set b = default where a = 1; +select * from default_v; + b | a +---------------+--- + table default | 1 +(1 row) + +alter view default_v alter column b set default 'view default'; +update default_v set b = default where a = 1; +select * from default_v; + b | a +--------------+--- + view default | 1 +(1 row) + \c regression drop database updatable_views_db; diff --git a/src/test/regress/sql/multi_update.sql b/src/test/regress/sql/multi_update.sql index 7e9f9b583..779663810 100644 --- a/src/test/regress/sql/multi_update.sql +++ b/src/test/regress/sql/multi_update.sql @@ -593,6 +593,11 @@ update t_t_mutil_t1 a,multiview2 b,t_t_mutil_t3 c set a.col2 = 4, b.col2 = 5,c.c select * from t_t_mutil_t1; select * from t_t_mutil_t2; select * from t_t_mutil_t3; +alter table t_t_mutil_t1 alter column col2 set default 999; +update t_t_mutil_t1 a,multiview2 b,t_t_mutil_t3 c set a.col2 = default, b.col2 = 5,c.col2 = 6 where a.col1 = b.col1 and a.col1 = c.col1; +select * from t_t_mutil_t1; +select * from t_t_mutil_t2; +select * from t_t_mutil_t3; -- left join update multiview1 a left join multiview2 b on a.col1=b.col1 set a.col2=7,b.col2=8; select * from t_t_mutil_t1; diff --git a/src/test/regress/sql/updatable_views.sql b/src/test/regress/sql/updatable_views.sql index f97fce2c9..84264efcf 100644 --- a/src/test/regress/sql/updatable_views.sql +++ b/src/test/regress/sql/updatable_views.sql @@ -1264,5 +1264,16 @@ set enable_bitmapscan = off; explain (costs off) UPDATE ro_view1 SET a = a + b; UPDATE ro_view1 SET a = a + b; +create table default_t (a int, b varchar default 'table default'); +insert into default_t values (1, 'dec'); +create view default_v as select b, a from default_t; + +update default_v set b = default where a = 1; +select * from default_v; + +alter view default_v alter column b set default 'view default'; +update default_v set b = default where a = 1; +select * from default_v; + \c regression drop database updatable_views_db; \ No newline at end of file From 3bb76dc41fc8f0ed92d3378d6525ec78f5b3fa8e Mon Sep 17 00:00:00 2001 From: pulsar Date: Tue, 18 Jul 2023 10:16:23 +0800 Subject: [PATCH 026/304] =?UTF-8?q?=E6=B3=A8=E9=87=8A=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E4=B8=BA=E8=8B=B1=E6=96=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/parser/analyze.cpp | 2 +- src/common/backend/utils/mmgr/portalmem.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/backend/parser/analyze.cpp b/src/common/backend/parser/analyze.cpp index a444c2236..a2a5b52c4 100644 --- a/src/common/backend/parser/analyze.cpp +++ b/src/common/backend/parser/analyze.cpp @@ -4429,9 +4429,9 @@ static Query* transformDeclareCursorStmt(ParseState* pstate, DeclareCursorStmt* ERROR, (errcode(ERRCODE_INVALID_CURSOR_DEFINITION), errmsg("cannot specify both SCROLL and NO SCROLL"))); } - /*除WITH HOLD游标外，根据DeclareCursorName对使用的row type形成依赖*/ PG_TRY(); { + /* according to DeclareCursorName to form a dependency on the used ROW type */ if (!(stmt->options & CURSOR_OPT_HOLD)) { u_sess->analyze_cxt.DeclareCursorName = stmt->portalname; } diff --git a/src/common/backend/utils/mmgr/portalmem.cpp b/src/common/backend/utils/mmgr/portalmem.cpp index a21a12148..53780e8da 100755 --- a/src/common/backend/utils/mmgr/portalmem.cpp +++ b/src/common/backend/utils/mmgr/portalmem.cpp @@ -1403,7 +1403,7 @@ HoldPinnedPortals(bool is_rollback) } } -/*解除游标与row type类型的依赖关系*/ +/* Release the dependency between CURSOR and ROW type */ static void CursorRecordTypeUnbind(const char* portal_name) { ListCell* cell = NULL; From 3842a723997130d97b5e001108d38bb8a5ed9482 Mon Sep 17 00:00:00 2001 From: Lamaric Date: Tue, 6 Jun 2023 19:23:10 +0800 Subject: [PATCH 027/304] =?UTF-8?q?[bugfix]=20pg=5Ffree=5Fremain=5Fsegment?= =?UTF-8?q?=20=E5=87=BD=E6=95=B0=E6=9C=AA=E5=A4=84=E7=90=86=E8=B4=9F?= =?UTF-8?q?=E6=95=B0=E5=85=A5=E5=8F=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/utils/adt/pgstatfuncs.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/common/backend/utils/adt/pgstatfuncs.cpp b/src/common/backend/utils/adt/pgstatfuncs.cpp index ad68260ce..7fcf17ee2 100644 --- a/src/common/backend/utils/adt/pgstatfuncs.cpp +++ b/src/common/backend/utils/adt/pgstatfuncs.cpp @@ -1818,9 +1818,9 @@ static void pgOutputRemainInfoToFile(RemainSegsCtx* remainSegsCtx) Datum pg_free_remain_segment(PG_FUNCTION_ARGS) { - Oid spaceId = PG_GETARG_OID(0); - Oid dbId = PG_GETARG_OID(1); - Oid segmentId = PG_GETARG_OID(2); + int32 spaceId = PG_GETARG_INT32(0); + int32 dbId = PG_GETARG_INT32(1); + int32 segmentId = PG_GETARG_INT32(2); RemainSegsCtx* remainSegsCtx = (RemainSegsCtx *)palloc(sizeof(RemainSegsCtx)); remainSegsCtx->remainSegsBuf = NULL; @@ -1828,7 +1828,7 @@ Datum pg_free_remain_segment(PG_FUNCTION_ARGS) ReadRemainSegsFileForFunc(remainSegsCtx); - int segIdx = pgCheckRemainSegment(remainSegsCtx, spaceId, dbId, segmentId); + int segIdx = pgCheckRemainSegment(remainSegsCtx, (Oid)spaceId, (Oid)dbId, (Oid)segmentId); if (segIdx >= 0 && (uint32)segIdx < remainSegsCtx->remainSegsNum) { pgDoFreeRemainSegment(remainSegsCtx, segIdx); @@ -1836,7 +1836,7 @@ Datum pg_free_remain_segment(PG_FUNCTION_ARGS) pgOutputRemainInfoToFile(remainSegsCtx); } else { - ereport(ERROR, (errmsg("Input segment [%u, %u, %u] is not remained segment!", spaceId, dbId, segmentId))); + ereport(ERROR, (errmsg("Input segment [%d, %d, %d] is not remained segment!", spaceId, dbId, segmentId))); } if (remainSegsCtx->remainSegsBuf != NULL) { From 7960232ba0e4b5dda4cb97edafee796164da3176 Mon Sep 17 00:00:00 2001 From: Lamaric Date: Tue, 18 Jul 2023 15:07:33 +0800 Subject: [PATCH 028/304] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=8F=82=E6=95=B0?= =?UTF-8?q?=E5=90=88=E6=B3=95=E6=80=A7=E6=A0=A1=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/utils/adt/pgstatfuncs.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/common/backend/utils/adt/pgstatfuncs.cpp b/src/common/backend/utils/adt/pgstatfuncs.cpp index 7fcf17ee2..dcd9e61e2 100644 --- a/src/common/backend/utils/adt/pgstatfuncs.cpp +++ b/src/common/backend/utils/adt/pgstatfuncs.cpp @@ -1822,6 +1822,11 @@ Datum pg_free_remain_segment(PG_FUNCTION_ARGS) int32 dbId = PG_GETARG_INT32(1); int32 segmentId = PG_GETARG_INT32(2); + if (spaceId < 0 || dbId < 0 || segmentId < 0) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("Input segment [%d, %d, %d] is not valid segment!", spaceId, dbId, segmentId))); + } + RemainSegsCtx* remainSegsCtx = (RemainSegsCtx *)palloc(sizeof(RemainSegsCtx)); remainSegsCtx->remainSegsBuf = NULL; remainSegsCtx->remainSegsNum = 0; From e13e3d09bc1b868e3a2f8bdb9ddce7464c0d3488 Mon Sep 17 00:00:00 2001 From: pulsar Date: Tue, 18 Jul 2023 15:13:57 +0800 Subject: [PATCH 029/304] =?UTF-8?q?=E8=A1=A5=E5=85=85=E7=94=A8=E4=BE=8B?= =?UTF-8?q?=E6=9C=9F=E6=9C=9B=E7=BB=93=E6=9E=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../expected/plpgsql_cursor_rowtype.out | 114 ++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/src/test/regress/expected/plpgsql_cursor_rowtype.out b/src/test/regress/expected/plpgsql_cursor_rowtype.out index 31a2e3422..1981e9116 100644 --- a/src/test/regress/expected/plpgsql_cursor_rowtype.out +++ b/src/test/regress/expected/plpgsql_cursor_rowtype.out @@ -845,6 +845,120 @@ INFO: aa set behavior_compat_options=''; drop procedure check_compile; +--游标依赖row type，后续alter type +create type foo as (a int, b int); +--游标依赖type，alter type报错 +begin; +declare c cursor for select (i,2^30)::foo from generate_series(1,10) i; +fetch c; + row +---------------- + (1,1073741824) +(1 row) + +fetch c; + row +---------------- + (2,1073741824) +(1 row) + +alter type foo alter attribute b type text;--error +ERROR: cannot ALTER TABLE "foo" because it is being used by active queries in this session +end; +--第二次开始从缓存中获取type +begin; +cursor c for select (i,2^30)::foo from generate_series(1,10) i; +fetch c; + row +---------------- + (1,1073741824) +(1 row) + +fetch c; + row +---------------- + (2,1073741824) +(1 row) + +alter type foo alter attribute b type text;--error +ERROR: cannot ALTER TABLE "foo" because it is being used by active queries in this session +end; +--close后，可以成功alter +begin; +declare c cursor for select (i,2^30)::foo from generate_series(1,10) i; +fetch c; + row +---------------- + (1,1073741824) +(1 row) + +fetch c; + row +---------------- + (2,1073741824) +(1 row) + +close c; +alter type foo alter attribute b type text;--success +declare c cursor for select (i,2^30)::foo from generate_series(1,10) i; +fetch c; + row +---------------- + (1,1073741824) +(1 row) + +fetch c; + row +---------------- + (2,1073741824) +(1 row) + +rollback; +begin; +cursor c for select (i,2^30)::foo from generate_series(1,10) i; +close c; +alter type foo alter attribute b type text;--success +end; +--多个游标依赖，只关闭一个 +begin; +cursor c1 for select (i,2^30)::foo from generate_series(1,10) i; +cursor c2 for select (i,2^30)::foo from generate_series(1,10) i; +close c1; +alter type foo alter attribute b type text;--error +ERROR: cannot ALTER TABLE "foo" because it is being used by active queries in this session +end; +--多个游标依赖，都关闭 +begin; +cursor c1 for select (i,2^30)::foo from generate_series(1,10) i; +cursor c2 for select (i,2^30)::foo from generate_series(1,10) i; +close c1; +close c2; +alter type foo alter attribute b type text;--success +end; +--WITH HOLD游标，事务结束继续保留 +begin; +cursor c3 WITH HOLD for select (i,2^30)::foo from generate_series(1,10) i; +fetch c3; + row +---------------- + (1,1073741824) +(1 row) + +end; +fetch c3; + row +---------------- + (2,1073741824) +(1 row) + +alter type foo alter attribute b type text;--success +fetch c3; + row +---------------- + (3,1073741824) +(1 row) + +close c3; ---- clean ---- drop package pck1; NOTICE: drop cascades to function plpgsql_cursor_rowtype.p1() From be0c54073d08ae1e1265e5bdb9ecf1870d331f36 Mon Sep 17 00:00:00 2001 From: luozihao <1165977584@qq.com> Date: Mon, 17 Jul 2023 21:09:25 +0800 Subject: [PATCH 030/304] =?UTF-8?q?=E4=B8=BA=E5=85=B6=E4=BB=96=E7=B1=BB?= =?UTF-8?q?=E5=9E=8B=E8=BD=AC=E5=8C=96=E4=B8=BAinterval=E6=B7=BB=E5=8A=A0t?= =?UTF-8?q?ypmod=E5=85=A5=E5=8F=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/catalog/builtin_funcs.ini | 15 + src/common/backend/catalog/system_views.sql | 12 +- src/common/backend/utils/adt/float.cpp | 10 + src/common/backend/utils/adt/int.cpp | 27 ++ src/common/backend/utils/adt/numeric.cpp | 39 ++ src/common/backend/utils/adt/varlena.cpp | 8 + src/common/backend/utils/init/globals.cpp | 2 +- src/include/catalog/pg_cast.h | 13 +- .../rollback-post_catalog_maindb_92_905.sql | 35 ++ .../rollback-post_catalog_otherdb_92_905.sql | 35 ++ .../upgrade-post_catalog_maindb_92_905.sql | 52 +++ .../upgrade-post_catalog_otherdb_92_905.sql | 52 +++ src/include/utils/builtins.h | 6 + .../expected/decode_compatible_with_o.out | 12 +- src/test/regress/expected/event.out | 2 +- src/test/regress/expected/interval.out | 385 ++++++++++++++++++ src/test/regress/sql/interval.sql | 69 ++++ 17 files changed, 755 insertions(+), 19 deletions(-) create mode 100644 src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback-post_catalog_maindb_92_905.sql create mode 100644 src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback-post_catalog_otherdb_92_905.sql create mode 100644 src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_905.sql create mode 100644 src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_905.sql diff --git a/src/common/backend/catalog/builtin_funcs.ini b/src/common/backend/catalog/builtin_funcs.ini index a2396fd20..ef8d3f4de 100755 --- a/src/common/backend/catalog/builtin_funcs.ini +++ b/src/common/backend/catalog/builtin_funcs.ini @@ -2926,6 +2926,10 @@ "float8_interval", 1, AddBuiltinFunc(_0(4229), _1("float8_interval"), _2(1), _3(true), _4(false), _5(float8_interval), _6(1186), _7(PG_CATALOG_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(0), _12(0), _13(0), _14(false), _15(false), _16(false), _17(false), _18('i'), _19(0), _20(1, 701), _21(NULL), _22(NULL), _23(NULL), _24(NULL), _25("float8_interval"), _26(NULL), _27(NULL), _28(NULL), _29(0), _30(false), _31(NULL), _32(false), _33(NULL), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)) ), + AddFuncGroup( + "float8_to_interval", 1, + AddBuiltinFunc(_0(4230), _1("float8_to_interval"), _2(2), _3(true), _4(false), _5(float8_to_interval), _6(1186), _7(PG_CATALOG_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(0), _12(0), _13(0), _14(false), _15(false), _16(false), _17(false), _18('i'), _19(0), _20(2, 701, 23), _21(NULL), _22(NULL), _23(NULL), _24(NULL), _25("float8_to_interval"), _26(NULL), _27(NULL), _28(NULL), _29(0), _30(false), _31(NULL), _32(false), _33(NULL), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)) + ), AddFuncGroup( "float8_list_agg_noarg2_transfn", 1, AddBuiltinFunc(_0(3573), _1("float8_list_agg_noarg2_transfn"), _2(2), _3(false), _4(false), _5(float8_list_agg_noarg2_transfn), _6(2281), _7(PG_CATALOG_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(0), _12(0), _13(0), _14(false), _15(false), _16(false), _17(false), _18('i'), _19(0), _20(2, 2281, 701), _21(NULL), _22(NULL), _23(NULL), _24(NULL), _25("float8_list_agg_noarg2_transfn"), _26(NULL), _27(NULL), _28(NULL), _29(0), _30(false), _31(NULL), _32(false), _33(NULL), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)) @@ -7515,6 +7519,13 @@ "numtodsinterval", 1, AddBuiltinFunc(_0(3172), _1("numtodsinterval"), _2(2), _3(true), _4(false), _5(numtodsinterval), _6(1186), _7(PG_CATALOG_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(0), _12(0), _13(0), _14(false), _15(false), _16(false), _17(false), _18('i'), _19(0), _20(2, 1700, 25), _21(NULL), _22(NULL), _23(NULL), _24(NULL), _25("numtodsinterval"), _26("-"), _27(NULL), _28(NULL), _29(0), _30(false), _31(NULL), _32(false), _33(NULL), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)) ), + AddFuncGroup( + "num_to_interval", 4, + AddBuiltinFunc(_0(4223), _1("num_to_interval"), _2(2), _3(true), _4(false), _5(int1_to_interval), _6(1186), _7(PG_CATALOG_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(0), _12(0), _13(0), _14(false), _15(false), _16(false), _17(false), _18('i'), _19(0), _20(2, 5545, 23), _21(NULL), _22(NULL), _23(NULL), _24(NULL), _25("int1_to_interval"), _26("-"), _27(NULL), _28(NULL), _29(0), _30(false), _31(NULL), _32(false), _33(NULL), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)), + AddBuiltinFunc(_0(4224), _1("num_to_interval"), _2(2), _3(true), _4(false), _5(int2_to_interval), _6(1186), _7(PG_CATALOG_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(0), _12(0), _13(0), _14(false), _15(false), _16(false), _17(false), _18('i'), _19(0), _20(2, 21, 23), _21(NULL), _22(NULL), _23(NULL), _24(NULL), _25("int2_to_interval"), _26("-"), _27(NULL), _28(NULL), _29(0), _30(false), _31(NULL), _32(false), _33(NULL), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)), + AddBuiltinFunc(_0(4225), _1("num_to_interval"), _2(2), _3(true), _4(false), _5(int4_to_interval), _6(1186), _7(PG_CATALOG_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(0), _12(0), _13(0), _14(false), _15(false), _16(false), _17(false), _18('i'), _19(0), _20(2, 23, 23), _21(NULL), _22(NULL), _23(NULL), _24(NULL), _25("int4_to_interval"), _26("-"), _27(NULL), _28(NULL), _29(0), _30(false), _31(NULL), _32(false), _33(NULL), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)), + AddBuiltinFunc(_0(4226), _1("num_to_interval"), _2(2), _3(true), _4(false), _5(numeric_to_interval), _6(1186), _7(PG_CATALOG_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(0), _12(0), _13(0), _14(false), _15(false), _16(false), _17(false), _18('i'), _19(0), _20(2, 1700, 23), _21(NULL), _22(NULL), _23(NULL), _24(NULL), _25("numeric_to_interval"), _26("-"), _27(NULL), _28(NULL), _29(0), _30(false), _31(NULL), _32(false), _33(NULL), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)) + ), AddFuncGroup( "nvarchar2", 1, AddBuiltinFunc(_0(3961), _1("nvarchar2"), _2(3), _3(true), _4(false), _5(nvarchar2), _6(3969), _7(PG_CATALOG_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(0), _12(0), _13(3097), _14(false), _15(false), _16(false), _17(false), _18('i'), _19(0), _20(3, 3969, 23, 16), _21(NULL), _22(NULL), _23(NULL), _24(NULL), _25("nvarchar2"), _26(NULL), _27(NULL), _28(NULL), _29(0), _30(false), _31(NULL), _32(false), _33(NULL), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)) @@ -11199,6 +11210,10 @@ AddFuncGroup( "text_int8", 1, AddBuiltinFunc(_0(4191), _1("text_int8"), _2(1), _3(true), _4(false), _5(text_int8), _6(20), _7(PG_CATALOG_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(0), _12(0), _13(0), _14(false), _15(false), _16(false), _17(false), _18('i'), _19(0), _20(1, 25), _21(NULL), _22(NULL), _23(NULL), _24(NULL), _25("text_int8"), _26(NULL), _27(NULL), _28(NULL), _29(0), _30(false), _31(NULL), _32(false), _33(NULL), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)) ), + AddFuncGroup( + "text_interval", 1, + AddBuiltinFunc(_0(4211), _1("text_interval"), _2(2), _3(true), _4(false), _5(text_interval), _6(1186), _7(PG_CATALOG_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(0), _12(0), _13(0), _14(false), _15(false), _16(false), _17(false), _18('s'), _19(0), _20(2, 25, 23), _21(NULL), _22(NULL), _23(NULL), _24(NULL), _25("text_interval"), _26(NULL), _27(NULL), _28(NULL), _29(0), _30(false), _31(true), _32(false), _33(NULL), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)) + ), AddFuncGroup( "text_larger", 1, AddBuiltinFunc(_0(458), _1("text_larger"), _2(2), _3(true), _4(false), _5(text_larger), _6(25), _7(PG_CATALOG_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(0), _12(0), _13(0), _14(false), _15(false), _16(false), _17(false), _18('i'), _19(0), _20(2, 25, 25), _21(NULL), _22(NULL), _23(NULL), _24(NULL), _25("text_larger"), _26(NULL), _27(NULL), _28(NULL), _29(0), _30(false), _31(NULL), _32(false), _33("larger of two"), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)) diff --git a/src/common/backend/catalog/system_views.sql b/src/common/backend/catalog/system_views.sql index 156819136..41d9f8cc1 100644 --- a/src/common/backend/catalog/system_views.sql +++ b/src/common/backend/catalog/system_views.sql @@ -2199,18 +2199,18 @@ LANGUAGE SQL IMMUTABLE STRICT NOT FENCED; CREATE CAST (INTERVAL AS VARCHAR2) WITH FUNCTION pg_catalog.TO_VARCHAR2(INTERVAL) AS IMPLICIT; /* char,varchar2 to interval */ -CREATE OR REPLACE FUNCTION pg_catalog.TO_INTERVAL(BPCHAR) +CREATE OR REPLACE FUNCTION pg_catalog.TO_INTERVAL(BPCHAR, int) RETURNS INTERVAL -AS $$ select pg_catalog.interval_in(pg_catalog.bpcharout($1), 0::Oid, -1) $$ +AS $$ select pg_catalog.interval_in(pg_catalog.bpcharout($1), 0::Oid, $2) $$ LANGUAGE SQL IMMUTABLE STRICT NOT FENCED; -CREATE OR REPLACE FUNCTION pg_catalog.TO_INTERVAL(VARCHAR2) +CREATE OR REPLACE FUNCTION pg_catalog.TO_INTERVAL(VARCHAR2, int) RETURNS INTERVAL -AS $$ select pg_catalog.interval_in(pg_catalog.varcharout($1), 0::Oid, -1) $$ +AS $$ select pg_catalog.interval_in(pg_catalog.varcharout($1), 0::Oid, $2) $$ LANGUAGE SQL IMMUTABLE STRICT NOT FENCED; -CREATE CAST (BPCHAR AS INTERVAL) WITH FUNCTION pg_catalog.TO_INTERVAL(BPCHAR) AS IMPLICIT; -CREATE CAST (VARCHAR2 AS INTERVAL) WITH FUNCTION pg_catalog.TO_INTERVAL(VARCHAR2) AS IMPLICIT; +CREATE CAST (BPCHAR AS INTERVAL) WITH FUNCTION pg_catalog.TO_INTERVAL(BPCHAR, int) AS IMPLICIT; +CREATE CAST (VARCHAR2 AS INTERVAL) WITH FUNCTION pg_catalog.TO_INTERVAL(VARCHAR2, int) AS IMPLICIT; /* raw to varchar2 */ CREATE CAST (RAW AS VARCHAR2) WITH FUNCTION pg_catalog.rawtohex(RAW) AS IMPLICIT; diff --git a/src/common/backend/utils/adt/float.cpp b/src/common/backend/utils/adt/float.cpp index c0bea9df6..1d41498ea 100644 --- a/src/common/backend/utils/adt/float.cpp +++ b/src/common/backend/utils/adt/float.cpp @@ -2914,3 +2914,13 @@ Datum float8_interval(PG_FUNCTION_ARGS) Datum val = PG_GETARG_DATUM(0); return DirectFunctionCall1(numeric_interval, DirectFunctionCall1(float8_numeric, val)); } + +/*convert from float8 to interval*/ +Datum float8_to_interval(PG_FUNCTION_ARGS) +{ + Datum val = PG_GETARG_DATUM(0); + int32 typmod = PG_GETARG_INT32(1); + return DirectFunctionCall2(numeric_to_interval, + DirectFunctionCall1(float8_numeric, val), + Int32GetDatum(typmod)); +} diff --git a/src/common/backend/utils/adt/int.cpp b/src/common/backend/utils/adt/int.cpp index bc854db1d..43df03da2 100644 --- a/src/common/backend/utils/adt/int.cpp +++ b/src/common/backend/utils/adt/int.cpp @@ -1523,6 +1523,33 @@ Datum int4_interval(PG_FUNCTION_ARGS) PG_RETURN_DATUM(DirectFunctionCall1(numeric_interval, DirectFunctionCall1(int4_numeric, PG_GETARG_DATUM(0)))); } +/* Cast int1 -> interval by typmod */ +Datum int1_to_interval(PG_FUNCTION_ARGS) +{ + int32 typmod = PG_GETARG_INT32(1); + PG_RETURN_DATUM(DirectFunctionCall2(numeric_to_interval, + DirectFunctionCall1(int1_numeric, PG_GETARG_DATUM(0)), + Int32GetDatum(typmod))); +} + +/* Cast int2 -> interval by typmod */ +Datum int2_to_interval(PG_FUNCTION_ARGS) +{ + int32 typmod = PG_GETARG_INT32(1); + PG_RETURN_DATUM(DirectFunctionCall2(numeric_to_interval, + DirectFunctionCall1(int2_numeric, PG_GETARG_DATUM(0)), + Int32GetDatum(typmod))); +} + +/* Cast int4 -> interval by typmod */ +Datum int4_to_interval(PG_FUNCTION_ARGS) +{ + int32 typmod = PG_GETARG_INT32(1); + PG_RETURN_DATUM(DirectFunctionCall2(numeric_to_interval, + DirectFunctionCall1(int4_numeric, PG_GETARG_DATUM(0)), + Int32GetDatum(typmod))); +} + /* OPERATE INT1 */ Datum int1um(PG_FUNCTION_ARGS) { diff --git a/src/common/backend/utils/adt/numeric.cpp b/src/common/backend/utils/adt/numeric.cpp index 6412ff70f..e6ebf51bd 100644 --- a/src/common/backend/utils/adt/numeric.cpp +++ b/src/common/backend/utils/adt/numeric.cpp @@ -7952,6 +7952,45 @@ Datum numeric_interval(PG_FUNCTION_ARGS) CHECK_RETNULL_RETURN_DATUM(result); } +/* Convert numeric to interval by typmod */ +Datum numeric_to_interval(PG_FUNCTION_ARGS) +{ + Datum num = PG_GETARG_DATUM(0); + int32 typmod = PG_GETARG_INT32(1); + Oid collation = PG_GET_COLLATION(); + Datum result; + StringInfoData str; + errno_t errorno = 0; + int str_len; + char* buf = NULL; + char* cp = NULL; + + CHECK_RETNULL_INIT(); + + initStringInfo(&str); + appendStringInfoString(&str, DatumGetCString(CHECK_RETNULL_CALL1(numeric_out_with_zero, collation, num))); + cp = str.data; + + if (*cp == '.' || (*cp == '-' && *(cp + 1) == '.')) { + str_len = str.len + 2; + buf = (char*)palloc0(str_len); + if (*cp == '.') { + errorno = snprintf_s(buf, str_len, str_len - 1, "0.%s", cp + 1); + } else { + errorno = snprintf_s(buf, str_len, str_len - 1, "-0.%s", cp + 2); + } + securec_check_ss(errorno, "\0", "\0"); + resetStringInfo(&str); + appendStringInfoString(&str, buf); + pfree_ext(buf); + } + + result = CHECK_RETNULL_CALL3( + interval_in, collation, CStringGetDatum(str.data), ObjectIdGetDatum(InvalidOid), Int32GetDatum(typmod)); + pfree_ext(str.data); + CHECK_RETNULL_RETURN_DATUM(result); +} + ScalarVector* vnumeric_sum(PG_FUNCTION_ARGS) { ScalarVector* pVector = (ScalarVector*)PG_GETARG_DATUM(0); diff --git a/src/common/backend/utils/adt/varlena.cpp b/src/common/backend/utils/adt/varlena.cpp index 0b929a3ab..9172a5959 100644 --- a/src/common/backend/utils/adt/varlena.cpp +++ b/src/common/backend/utils/adt/varlena.cpp @@ -7330,3 +7330,11 @@ Datum btvarstrequalimage(PG_FUNCTION_ARGS) else PG_RETURN_BOOL(false); } + +Datum text_interval(PG_FUNCTION_ARGS) +{ + char* input = TextDatumGetCString(PG_GETARG_TEXT_P(0)); + int32 typmod = PG_GETARG_INT32(1); + return DirectFunctionCall3(interval_in, CStringGetDatum(input), ObjectIdGetDatum(InvalidOid), + Int32GetDatum(typmod)); +} diff --git a/src/common/backend/utils/init/globals.cpp b/src/common/backend/utils/init/globals.cpp index 525cd978b..f805918a8 100644 --- a/src/common/backend/utils/init/globals.cpp +++ b/src/common/backend/utils/init/globals.cpp @@ -75,7 +75,7 @@ bool will_shutdown = false; * NEXT | 92899 | ? | ? * ********************************************/ -const uint32 GRAND_VERSION_NUM = 92904; +const uint32 GRAND_VERSION_NUM = 92905; /******************************************** * 2.VERSION NUM FOR EACH FEATURE diff --git a/src/include/catalog/pg_cast.h b/src/include/catalog/pg_cast.h index c789821d9..e2532011f 100644 --- a/src/include/catalog/pg_cast.h +++ b/src/include/catalog/pg_cast.h @@ -202,19 +202,22 @@ DATA(insert ( 16 1700 6433 i f _null_)); DATA(insert ( 23 1042 3192 i f _null_)); /* numeric ->interval */ -DATA(insert ( 1700 1186 3842 i f _null_)); +DATA(insert ( 1700 1186 4226 i f _null_)); /* float8-> interval */ -DATA(insert ( 701 1186 4229 i f _null_)); +DATA(insert ( 701 1186 4230 i f _null_)); /* int1 ->interval */ -DATA(insert ( 5545 1186 3189 i f _null_)); +DATA(insert ( 5545 1186 4223 i f _null_)); /* int2 ->interval */ -DATA(insert ( 21 1186 3190 i f _null_)); +DATA(insert ( 21 1186 4224 i f _null_)); /* int4 ->interval */ -DATA(insert ( 23 1186 3191 i f _null_)); +DATA(insert ( 23 1186 4225 i f _null_)); + +/* text ->interval */ +DATA(insert ( 25 1186 4211 i f _null_)); /* * OID category: allow implicit conversion from any integral type (including diff --git a/src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback-post_catalog_maindb_92_905.sql b/src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback-post_catalog_maindb_92_905.sql new file mode 100644 index 000000000..e903998ac --- /dev/null +++ b/src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback-post_catalog_maindb_92_905.sql @@ -0,0 +1,35 @@ +DROP CAST IF EXISTS (INT1 AS INTERVAL); +DROP CAST IF EXISTS (INT2 AS INTERVAL); +DROP CAST IF EXISTS (INT4 AS INTERVAL); +DROP CAST IF EXISTS (FLOAT8 AS INTERVAL); +DROP CAST IF EXISTS (NUMERIC AS INTERVAL); +DROP CAST IF EXISTS (TEXT AS INTERVAL); +DROP CAST IF EXISTS (BPCHAR AS INTERVAL); +DROP CAST IF EXISTS (VARCHAR2 AS INTERVAL); + +DROP FUNCTION IF EXISTS pg_catalog.num_to_interval(int1, int4) CASCADE; +DROP FUNCTION IF EXISTS pg_catalog.num_to_interval(int2, int4) CASCADE; +DROP FUNCTION IF EXISTS pg_catalog.num_to_interval(int4, int4) CASCADE; +DROP FUNCTION IF EXISTS pg_catalog.float8_to_interval(float8, int4) CASCADE; +DROP FUNCTION IF EXISTS pg_catalog.num_to_interval(numeric, int4) CASCADE; +DROP FUNCTION IF EXISTS pg_catalog.text_interval(text, int4) CASCADE; + +DROP FUNCTION IF EXISTS pg_catalog.TO_INTERVAL(BPCHAR, int); +CREATE OR REPLACE FUNCTION pg_catalog.TO_INTERVAL(BPCHAR) +RETURNS INTERVAL +AS $$ select pg_catalog.interval_in(pg_catalog.bpcharout($1), 0::Oid, -1) $$ +LANGUAGE SQL IMMUTABLE STRICT NOT FENCED; + +DROP FUNCTION IF EXISTS pg_catalog.TO_INTERVAL(VARCHAR2, int); +CREATE OR REPLACE FUNCTION pg_catalog.TO_INTERVAL(VARCHAR2) +RETURNS INTERVAL +AS $$ select pg_catalog.interval_in(pg_catalog.varcharout($1), 0::Oid, -1) $$ +LANGUAGE SQL IMMUTABLE STRICT NOT FENCED; + +CREATE CAST (INT1 AS INTERVAL) WITH FUNCTION pg_catalog.numtoday(INT1) AS IMPLICIT; +CREATE CAST (INT2 AS INTERVAL) WITH FUNCTION pg_catalog.numtoday(INT2) AS IMPLICIT; +CREATE CAST (INT4 AS INTERVAL) WITH FUNCTION pg_catalog.numtoday(INT4) AS IMPLICIT; +CREATE CAST (FLOAT8 AS INTERVAL) WITH FUNCTION pg_catalog.float8_interval(FLOAT8) AS IMPLICIT; +CREATE CAST (NUMERIC AS INTERVAL) WITH FUNCTION pg_catalog.numtoday(NUMERIC) AS IMPLICIT; +CREATE CAST (BPCHAR AS INTERVAL) WITH FUNCTION pg_catalog.TO_INTERVAL(BPCHAR) AS IMPLICIT; +CREATE CAST (VARCHAR2 AS INTERVAL) WITH FUNCTION pg_catalog.TO_INTERVAL(VARCHAR2) AS IMPLICIT; diff --git a/src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback-post_catalog_otherdb_92_905.sql b/src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback-post_catalog_otherdb_92_905.sql new file mode 100644 index 000000000..e903998ac --- /dev/null +++ b/src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback-post_catalog_otherdb_92_905.sql @@ -0,0 +1,35 @@ +DROP CAST IF EXISTS (INT1 AS INTERVAL); +DROP CAST IF EXISTS (INT2 AS INTERVAL); +DROP CAST IF EXISTS (INT4 AS INTERVAL); +DROP CAST IF EXISTS (FLOAT8 AS INTERVAL); +DROP CAST IF EXISTS (NUMERIC AS INTERVAL); +DROP CAST IF EXISTS (TEXT AS INTERVAL); +DROP CAST IF EXISTS (BPCHAR AS INTERVAL); +DROP CAST IF EXISTS (VARCHAR2 AS INTERVAL); + +DROP FUNCTION IF EXISTS pg_catalog.num_to_interval(int1, int4) CASCADE; +DROP FUNCTION IF EXISTS pg_catalog.num_to_interval(int2, int4) CASCADE; +DROP FUNCTION IF EXISTS pg_catalog.num_to_interval(int4, int4) CASCADE; +DROP FUNCTION IF EXISTS pg_catalog.float8_to_interval(float8, int4) CASCADE; +DROP FUNCTION IF EXISTS pg_catalog.num_to_interval(numeric, int4) CASCADE; +DROP FUNCTION IF EXISTS pg_catalog.text_interval(text, int4) CASCADE; + +DROP FUNCTION IF EXISTS pg_catalog.TO_INTERVAL(BPCHAR, int); +CREATE OR REPLACE FUNCTION pg_catalog.TO_INTERVAL(BPCHAR) +RETURNS INTERVAL +AS $$ select pg_catalog.interval_in(pg_catalog.bpcharout($1), 0::Oid, -1) $$ +LANGUAGE SQL IMMUTABLE STRICT NOT FENCED; + +DROP FUNCTION IF EXISTS pg_catalog.TO_INTERVAL(VARCHAR2, int); +CREATE OR REPLACE FUNCTION pg_catalog.TO_INTERVAL(VARCHAR2) +RETURNS INTERVAL +AS $$ select pg_catalog.interval_in(pg_catalog.varcharout($1), 0::Oid, -1) $$ +LANGUAGE SQL IMMUTABLE STRICT NOT FENCED; + +CREATE CAST (INT1 AS INTERVAL) WITH FUNCTION pg_catalog.numtoday(INT1) AS IMPLICIT; +CREATE CAST (INT2 AS INTERVAL) WITH FUNCTION pg_catalog.numtoday(INT2) AS IMPLICIT; +CREATE CAST (INT4 AS INTERVAL) WITH FUNCTION pg_catalog.numtoday(INT4) AS IMPLICIT; +CREATE CAST (FLOAT8 AS INTERVAL) WITH FUNCTION pg_catalog.float8_interval(FLOAT8) AS IMPLICIT; +CREATE CAST (NUMERIC AS INTERVAL) WITH FUNCTION pg_catalog.numtoday(NUMERIC) AS IMPLICIT; +CREATE CAST (BPCHAR AS INTERVAL) WITH FUNCTION pg_catalog.TO_INTERVAL(BPCHAR) AS IMPLICIT; +CREATE CAST (VARCHAR2 AS INTERVAL) WITH FUNCTION pg_catalog.TO_INTERVAL(VARCHAR2) AS IMPLICIT; diff --git a/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_905.sql b/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_905.sql new file mode 100644 index 000000000..788136747 --- /dev/null +++ b/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_905.sql @@ -0,0 +1,52 @@ +DROP CAST IF EXISTS (INT1 AS INTERVAL); +DROP CAST IF EXISTS (INT2 AS INTERVAL); +DROP CAST IF EXISTS (INT4 AS INTERVAL); +DROP CAST IF EXISTS (FLOAT8 AS INTERVAL); +DROP CAST IF EXISTS (NUMERIC AS INTERVAL); +DROP CAST IF EXISTS (BPCHAR AS INTERVAL); +DROP CAST IF EXISTS (VARCHAR2 AS INTERVAL); + +DROP FUNCTION IF EXISTS pg_catalog.num_to_interval(int1, int4) CASCADE; +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 4223; +CREATE FUNCTION pg_catalog.num_to_interval(int1, int4) RETURNS INTERVAL LANGUAGE INTERNAL IMMUTABLE AS 'int1_to_interval'; + +DROP FUNCTION IF EXISTS pg_catalog.num_to_interval(int2, int4) CASCADE; +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 4224; +CREATE FUNCTION pg_catalog.num_to_interval(int2, int4) RETURNS INTERVAL LANGUAGE INTERNAL IMMUTABLE AS 'int2_to_interval'; + +DROP FUNCTION IF EXISTS pg_catalog.num_to_interval(int4, int4) CASCADE; +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 4225; +CREATE FUNCTION pg_catalog.num_to_interval(int4, int4) RETURNS INTERVAL LANGUAGE INTERNAL IMMUTABLE AS 'int4_to_interval'; + +DROP FUNCTION IF EXISTS pg_catalog.float8_to_interval(float8, int4) CASCADE; +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 4230; +CREATE FUNCTION pg_catalog.float8_to_interval(float8, int4) RETURNS INTERVAL LANGUAGE INTERNAL IMMUTABLE AS 'float8_to_interval'; + +DROP FUNCTION IF EXISTS pg_catalog.num_to_interval(numeric, int4) CASCADE; +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 4226; +CREATE FUNCTION pg_catalog.num_to_interval(numeric, int4) RETURNS INTERVAL LANGUAGE INTERNAL IMMUTABLE AS 'numeric_to_interval'; + +DROP FUNCTION IF EXISTS pg_catalog.text_interval(text, int4) CASCADE; +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 4211; +CREATE FUNCTION pg_catalog.text_interval(text, int4) RETURNS INTERVAL LANGUAGE INTERNAL IMMUTABLE AS 'text_interval'; + +DROP FUNCTION IF EXISTS pg_catalog.TO_INTERVAL(BPCHAR); +CREATE OR REPLACE FUNCTION pg_catalog.TO_INTERVAL(BPCHAR, int) +RETURNS INTERVAL +AS $$ select pg_catalog.interval_in(pg_catalog.bpcharout($1), 0::Oid, $2) $$ +LANGUAGE SQL IMMUTABLE STRICT NOT FENCED; + +DROP FUNCTION IF EXISTS pg_catalog.TO_INTERVAL(VARCHAR2); +CREATE OR REPLACE FUNCTION pg_catalog.TO_INTERVAL(VARCHAR2, int) +RETURNS INTERVAL +AS $$ select pg_catalog.interval_in(pg_catalog.varcharout($1), 0::Oid, $2) $$ +LANGUAGE SQL IMMUTABLE STRICT NOT FENCED; + +CREATE CAST (INT1 AS INTERVAL) WITH FUNCTION pg_catalog.num_to_interval(INT1, int) AS IMPLICIT; +CREATE CAST (INT2 AS INTERVAL) WITH FUNCTION pg_catalog.num_to_interval(INT2, int) AS IMPLICIT; +CREATE CAST (INT4 AS INTERVAL) WITH FUNCTION pg_catalog.num_to_interval(INT4, int) AS IMPLICIT; +CREATE CAST (FLOAT8 AS INTERVAL) WITH FUNCTION pg_catalog.float8_to_interval(FLOAT8, int) AS IMPLICIT; +CREATE CAST (NUMERIC AS INTERVAL) WITH FUNCTION pg_catalog.num_to_interval(NUMERIC, int) AS IMPLICIT; +CREATE CAST (TEXT AS INTERVAL) WITH FUNCTION pg_catalog.TEXT_INTERVAL(TEXT, int) AS IMPLICIT; +CREATE CAST (BPCHAR AS INTERVAL) WITH FUNCTION pg_catalog.TO_INTERVAL(BPCHAR, int) AS IMPLICIT; +CREATE CAST (VARCHAR2 AS INTERVAL) WITH FUNCTION pg_catalog.TO_INTERVAL(VARCHAR2, int) AS IMPLICIT; diff --git a/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_905.sql b/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_905.sql new file mode 100644 index 000000000..788136747 --- /dev/null +++ b/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_905.sql @@ -0,0 +1,52 @@ +DROP CAST IF EXISTS (INT1 AS INTERVAL); +DROP CAST IF EXISTS (INT2 AS INTERVAL); +DROP CAST IF EXISTS (INT4 AS INTERVAL); +DROP CAST IF EXISTS (FLOAT8 AS INTERVAL); +DROP CAST IF EXISTS (NUMERIC AS INTERVAL); +DROP CAST IF EXISTS (BPCHAR AS INTERVAL); +DROP CAST IF EXISTS (VARCHAR2 AS INTERVAL); + +DROP FUNCTION IF EXISTS pg_catalog.num_to_interval(int1, int4) CASCADE; +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 4223; +CREATE FUNCTION pg_catalog.num_to_interval(int1, int4) RETURNS INTERVAL LANGUAGE INTERNAL IMMUTABLE AS 'int1_to_interval'; + +DROP FUNCTION IF EXISTS pg_catalog.num_to_interval(int2, int4) CASCADE; +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 4224; +CREATE FUNCTION pg_catalog.num_to_interval(int2, int4) RETURNS INTERVAL LANGUAGE INTERNAL IMMUTABLE AS 'int2_to_interval'; + +DROP FUNCTION IF EXISTS pg_catalog.num_to_interval(int4, int4) CASCADE; +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 4225; +CREATE FUNCTION pg_catalog.num_to_interval(int4, int4) RETURNS INTERVAL LANGUAGE INTERNAL IMMUTABLE AS 'int4_to_interval'; + +DROP FUNCTION IF EXISTS pg_catalog.float8_to_interval(float8, int4) CASCADE; +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 4230; +CREATE FUNCTION pg_catalog.float8_to_interval(float8, int4) RETURNS INTERVAL LANGUAGE INTERNAL IMMUTABLE AS 'float8_to_interval'; + +DROP FUNCTION IF EXISTS pg_catalog.num_to_interval(numeric, int4) CASCADE; +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 4226; +CREATE FUNCTION pg_catalog.num_to_interval(numeric, int4) RETURNS INTERVAL LANGUAGE INTERNAL IMMUTABLE AS 'numeric_to_interval'; + +DROP FUNCTION IF EXISTS pg_catalog.text_interval(text, int4) CASCADE; +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 4211; +CREATE FUNCTION pg_catalog.text_interval(text, int4) RETURNS INTERVAL LANGUAGE INTERNAL IMMUTABLE AS 'text_interval'; + +DROP FUNCTION IF EXISTS pg_catalog.TO_INTERVAL(BPCHAR); +CREATE OR REPLACE FUNCTION pg_catalog.TO_INTERVAL(BPCHAR, int) +RETURNS INTERVAL +AS $$ select pg_catalog.interval_in(pg_catalog.bpcharout($1), 0::Oid, $2) $$ +LANGUAGE SQL IMMUTABLE STRICT NOT FENCED; + +DROP FUNCTION IF EXISTS pg_catalog.TO_INTERVAL(VARCHAR2); +CREATE OR REPLACE FUNCTION pg_catalog.TO_INTERVAL(VARCHAR2, int) +RETURNS INTERVAL +AS $$ select pg_catalog.interval_in(pg_catalog.varcharout($1), 0::Oid, $2) $$ +LANGUAGE SQL IMMUTABLE STRICT NOT FENCED; + +CREATE CAST (INT1 AS INTERVAL) WITH FUNCTION pg_catalog.num_to_interval(INT1, int) AS IMPLICIT; +CREATE CAST (INT2 AS INTERVAL) WITH FUNCTION pg_catalog.num_to_interval(INT2, int) AS IMPLICIT; +CREATE CAST (INT4 AS INTERVAL) WITH FUNCTION pg_catalog.num_to_interval(INT4, int) AS IMPLICIT; +CREATE CAST (FLOAT8 AS INTERVAL) WITH FUNCTION pg_catalog.float8_to_interval(FLOAT8, int) AS IMPLICIT; +CREATE CAST (NUMERIC AS INTERVAL) WITH FUNCTION pg_catalog.num_to_interval(NUMERIC, int) AS IMPLICIT; +CREATE CAST (TEXT AS INTERVAL) WITH FUNCTION pg_catalog.TEXT_INTERVAL(TEXT, int) AS IMPLICIT; +CREATE CAST (BPCHAR AS INTERVAL) WITH FUNCTION pg_catalog.TO_INTERVAL(BPCHAR, int) AS IMPLICIT; +CREATE CAST (VARCHAR2 AS INTERVAL) WITH FUNCTION pg_catalog.TO_INTERVAL(VARCHAR2, int) AS IMPLICIT; diff --git a/src/include/utils/builtins.h b/src/include/utils/builtins.h index 553049adb..c01264ee0 100644 --- a/src/include/utils/builtins.h +++ b/src/include/utils/builtins.h @@ -1346,6 +1346,7 @@ extern Datum int1_numeric(PG_FUNCTION_ARGS); extern Datum numeric_int1(PG_FUNCTION_ARGS); extern Datum float8_numeric(PG_FUNCTION_ARGS); extern Datum float8_interval(PG_FUNCTION_ARGS); +extern Datum float8_to_interval(PG_FUNCTION_ARGS); extern Datum numeric_float8(PG_FUNCTION_ARGS); extern Datum numeric_float8_no_overflow(PG_FUNCTION_ARGS); extern Datum float4_numeric(PG_FUNCTION_ARGS); @@ -1397,9 +1398,13 @@ extern Datum width_bucket_numeric(PG_FUNCTION_ARGS); extern Datum hash_numeric(PG_FUNCTION_ARGS); extern Datum numtodsinterval(PG_FUNCTION_ARGS); extern Datum numeric_interval(PG_FUNCTION_ARGS); +extern Datum numeric_to_interval(PG_FUNCTION_ARGS); extern Datum int1_interval(PG_FUNCTION_ARGS); extern Datum int2_interval(PG_FUNCTION_ARGS); extern Datum int4_interval(PG_FUNCTION_ARGS); +extern Datum int1_to_interval(PG_FUNCTION_ARGS); +extern Datum int2_to_interval(PG_FUNCTION_ARGS); +extern Datum int4_to_interval(PG_FUNCTION_ARGS); /* ri_triggers.c */ extern Datum RI_FKey_check_ins(PG_FUNCTION_ARGS); @@ -1829,6 +1834,7 @@ extern Datum pg_show_replication_origin_status(PG_FUNCTION_ARGS); extern Datum btequalimage(PG_FUNCTION_ARGS); /* varlena.cpp */ extern Datum btvarstrequalimage(PG_FUNCTION_ARGS); +extern Datum text_interval(PG_FUNCTION_ARGS); /* pg_publication.cpp */ extern Datum pg_get_publication_tables(PG_FUNCTION_ARGS); diff --git a/src/test/regress/expected/decode_compatible_with_o.out b/src/test/regress/expected/decode_compatible_with_o.out index 07fcdd555..c0f7c1e35 100755 --- a/src/test/regress/expected/decode_compatible_with_o.out +++ b/src/test/regress/expected/decode_compatible_with_o.out @@ -4972,9 +4972,9 @@ select decode(c_reltime, c_nvarchar2, 'Conversion successfully!', 'Conversion fa (1 row) select decode(c_interval, c_text, 'Conversion successfully!', 'Conversion failed!') from tb_test; - case --------------------- - Conversion failed! + case +-------------------------- + Conversion successfully! (1 row) select decode(c_reltime, c_text, 'Conversion successfully!', 'Conversion failed!') from tb_test; @@ -5008,9 +5008,9 @@ select decode(c_nvarchar2, c_interval, 'Conversion successfully!', 'Conversion f (1 row) select decode(c_text, c_interval, 'Conversion successfully!', 'Conversion failed!') from tb_test; - case --------------------- - Conversion failed! + case +-------------------------- + Conversion successfully! (1 row) select decode(c_reltime, c_interval, 'Conversion successfully!', 'Conversion failed!') from tb_test; diff --git a/src/test/regress/expected/event.out b/src/test/regress/expected/event.out index b3946c9f3..809a9f2d4 100644 --- a/src/test/regress/expected/event.out +++ b/src/test/regress/expected/event.out @@ -289,7 +289,7 @@ select job_name, nspname from pg_job where dbname='event_b'; drop event if exists ee11; create event IF NOT EXISTS ee11 on schedule at now() + interval 666666666666666666666666666667 year + interval '00:00' minute to second disable do insert into t values(0); -ERROR: interval field value out of range: "666666666666666666666666666667 day" +ERROR: interval field value out of range: "666666666666666666666666666667" select pg_sleep(0.2); pg_sleep ---------- diff --git a/src/test/regress/expected/interval.out b/src/test/regress/expected/interval.out index f418ff6d8..fbe5b481a 100644 --- a/src/test/regress/expected/interval.out +++ b/src/test/regress/expected/interval.out @@ -925,3 +925,388 @@ SELECT '烦%￥' || CAST(NULL AS INTERVAL); 烦%￥ (1 row) +-- test about cast +select 15::int1::INTERVAL MINUTE; + interval +----------- + @ 15 mins +(1 row) + +select 15::int2::INTERVAL MINUTE; + interval +----------- + @ 15 mins +(1 row) + +select 15::int4::INTERVAL MINUTE; + interval +----------- + @ 15 mins +(1 row) + +select 15::float8::INTERVAL MINUTE; + interval +----------- + @ 15 mins +(1 row) + +select 15::numeric::INTERVAL MINUTE; + interval +----------- + @ 15 mins +(1 row) + +select 15::text::INTERVAL MINUTE; + interval +----------- + @ 15 mins +(1 row) + +select 15::varchar::INTERVAL MINUTE; + interval +----------- + @ 15 mins +(1 row) + +select 15::bpchar(2)::INTERVAL MINUTE; + interval +----------- + @ 15 mins +(1 row) + +select '15'::int1::INTERVAL MINUTE; + interval +----------- + @ 15 mins +(1 row) + +select '15'::int2::INTERVAL MINUTE; + interval +----------- + @ 15 mins +(1 row) + +select '15'::int4::INTERVAL MINUTE; + interval +----------- + @ 15 mins +(1 row) + +select '15'::float8::INTERVAL MINUTE; + interval +----------- + @ 15 mins +(1 row) + +select '15'::numeric::INTERVAL MINUTE; + interval +----------- + @ 15 mins +(1 row) + +select '15'::text::INTERVAL MINUTE; + interval +----------- + @ 15 mins +(1 row) + +select '15'::varchar::INTERVAL MINUTE; + interval +----------- + @ 15 mins +(1 row) + +select '15'::bpchar(2)::INTERVAL MINUTE; + interval +----------- + @ 15 mins +(1 row) + +select 15::int1::INTERVAL HOUR; + interval +------------ + @ 15 hours +(1 row) + +select 15::int2::INTERVAL HOUR; + interval +------------ + @ 15 hours +(1 row) + +select 15::int4::INTERVAL HOUR; + interval +------------ + @ 15 hours +(1 row) + +select 15::float8::INTERVAL HOUR; + interval +------------ + @ 15 hours +(1 row) + +select 15::numeric::INTERVAL HOUR; + interval +------------ + @ 15 hours +(1 row) + +select 15::text::INTERVAL HOUR; + interval +------------ + @ 15 hours +(1 row) + +select 15::varchar::INTERVAL HOUR; + interval +------------ + @ 15 hours +(1 row) + +select 15::bpchar(2)::INTERVAL HOUR; + interval +------------ + @ 15 hours +(1 row) + +select '15'::int1::INTERVAL HOUR; + interval +------------ + @ 15 hours +(1 row) + +select '15'::int2::INTERVAL HOUR; + interval +------------ + @ 15 hours +(1 row) + +select '15'::int4::INTERVAL HOUR; + interval +------------ + @ 15 hours +(1 row) + +select '15'::float8::INTERVAL HOUR; + interval +------------ + @ 15 hours +(1 row) + +select '15'::numeric::INTERVAL HOUR; + interval +------------ + @ 15 hours +(1 row) + +select '15'::text::INTERVAL HOUR; + interval +------------ + @ 15 hours +(1 row) + +select '15'::varchar::INTERVAL HOUR; + interval +------------ + @ 15 hours +(1 row) + +select '15'::bpchar(2)::INTERVAL HOUR; + interval +------------ + @ 15 hours +(1 row) + +select 15::int1::INTERVAL ; + interval +----------- + @ 15 days +(1 row) + +select 15::int2::INTERVAL ; + interval +----------- + @ 15 days +(1 row) + +select 15::int4::INTERVAL ; + interval +----------- + @ 15 days +(1 row) + +select 15::float8::INTERVAL ; + interval +----------- + @ 15 days +(1 row) + +select 15::numeric::INTERVAL ; + interval +----------- + @ 15 days +(1 row) + +select 15::text::INTERVAL ; + interval +----------- + @ 15 days +(1 row) + +select 15::varchar::INTERVAL ; + interval +----------- + @ 15 days +(1 row) + +select 15::bpchar(2)::INTERVAL ; + interval +----------- + @ 15 days +(1 row) + +select '15'::int1::INTERVAL ; + interval +----------- + @ 15 days +(1 row) + +select '15'::int2::INTERVAL ; + interval +----------- + @ 15 days +(1 row) + +select '15'::int4::INTERVAL ; + interval +----------- + @ 15 days +(1 row) + +select '15'::float8::INTERVAL ; + interval +----------- + @ 15 days +(1 row) + +select '15'::numeric::INTERVAL ; + interval +----------- + @ 15 days +(1 row) + +select '15'::text::INTERVAL ; + interval +----------- + @ 15 days +(1 row) + +select '15'::varchar::INTERVAL ; + interval +----------- + @ 15 days +(1 row) + +select '15'::bpchar(2)::INTERVAL ; + interval +----------- + @ 15 days +(1 row) + +select 15::int1::INTERVAL YEAR; + interval +------------ + @ 15 years +(1 row) + +select 15::int2::INTERVAL YEAR; + interval +------------ + @ 15 years +(1 row) + +select 15::int4::INTERVAL YEAR; + interval +------------ + @ 15 years +(1 row) + +select 15::float8::INTERVAL YEAR; + interval +------------ + @ 15 years +(1 row) + +select 15::numeric::INTERVAL YEAR; + interval +------------ + @ 15 years +(1 row) + +select 15::text::INTERVAL YEAR; + interval +------------ + @ 15 years +(1 row) + +select 15::varchar::INTERVAL YEAR; + interval +------------ + @ 15 years +(1 row) + +select 15::bpchar(2)::INTERVAL YEAR; + interval +------------ + @ 15 years +(1 row) + +select '15'::int1::INTERVAL YEAR; + interval +------------ + @ 15 years +(1 row) + +select '15'::int2::INTERVAL YEAR; + interval +------------ + @ 15 years +(1 row) + +select '15'::int4::INTERVAL YEAR; + interval +------------ + @ 15 years +(1 row) + +select '15'::float8::INTERVAL YEAR; + interval +------------ + @ 15 years +(1 row) + +select '15'::numeric::INTERVAL YEAR; + interval +------------ + @ 15 years +(1 row) + +select '15'::text::INTERVAL YEAR; + interval +------------ + @ 15 years +(1 row) + +select '15'::varchar::INTERVAL YEAR; + interval +------------ + @ 15 years +(1 row) + +select '15'::bpchar(2)::INTERVAL YEAR; + interval +------------ + @ 15 years +(1 row) + diff --git a/src/test/regress/sql/interval.sql b/src/test/regress/sql/interval.sql index 7f3b7aea6..4b5f97bd1 100644 --- a/src/test/regress/sql/interval.sql +++ b/src/test/regress/sql/interval.sql @@ -299,3 +299,72 @@ select interval_hash('30 days'::interval) = interval_hash('1 month'::interval) a SELECT 'ABC' || CAST(NULL AS INTERVAL); SELECT '烦%￥' || CAST(NULL AS INTERVAL); + +-- test about cast +select 15::int1::INTERVAL MINUTE; +select 15::int2::INTERVAL MINUTE; +select 15::int4::INTERVAL MINUTE; +select 15::float8::INTERVAL MINUTE; +select 15::numeric::INTERVAL MINUTE; +select 15::text::INTERVAL MINUTE; +select 15::varchar::INTERVAL MINUTE; +select 15::bpchar(2)::INTERVAL MINUTE; +select '15'::int1::INTERVAL MINUTE; +select '15'::int2::INTERVAL MINUTE; +select '15'::int4::INTERVAL MINUTE; +select '15'::float8::INTERVAL MINUTE; +select '15'::numeric::INTERVAL MINUTE; +select '15'::text::INTERVAL MINUTE; +select '15'::varchar::INTERVAL MINUTE; +select '15'::bpchar(2)::INTERVAL MINUTE; + +select 15::int1::INTERVAL HOUR; +select 15::int2::INTERVAL HOUR; +select 15::int4::INTERVAL HOUR; +select 15::float8::INTERVAL HOUR; +select 15::numeric::INTERVAL HOUR; +select 15::text::INTERVAL HOUR; +select 15::varchar::INTERVAL HOUR; +select 15::bpchar(2)::INTERVAL HOUR; +select '15'::int1::INTERVAL HOUR; +select '15'::int2::INTERVAL HOUR; +select '15'::int4::INTERVAL HOUR; +select '15'::float8::INTERVAL HOUR; +select '15'::numeric::INTERVAL HOUR; +select '15'::text::INTERVAL HOUR; +select '15'::varchar::INTERVAL HOUR; +select '15'::bpchar(2)::INTERVAL HOUR; + +select 15::int1::INTERVAL ; +select 15::int2::INTERVAL ; +select 15::int4::INTERVAL ; +select 15::float8::INTERVAL ; +select 15::numeric::INTERVAL ; +select 15::text::INTERVAL ; +select 15::varchar::INTERVAL ; +select 15::bpchar(2)::INTERVAL ; +select '15'::int1::INTERVAL ; +select '15'::int2::INTERVAL ; +select '15'::int4::INTERVAL ; +select '15'::float8::INTERVAL ; +select '15'::numeric::INTERVAL ; +select '15'::text::INTERVAL ; +select '15'::varchar::INTERVAL ; +select '15'::bpchar(2)::INTERVAL ; + +select 15::int1::INTERVAL YEAR; +select 15::int2::INTERVAL YEAR; +select 15::int4::INTERVAL YEAR; +select 15::float8::INTERVAL YEAR; +select 15::numeric::INTERVAL YEAR; +select 15::text::INTERVAL YEAR; +select 15::varchar::INTERVAL YEAR; +select 15::bpchar(2)::INTERVAL YEAR; +select '15'::int1::INTERVAL YEAR; +select '15'::int2::INTERVAL YEAR; +select '15'::int4::INTERVAL YEAR; +select '15'::float8::INTERVAL YEAR; +select '15'::numeric::INTERVAL YEAR; +select '15'::text::INTERVAL YEAR; +select '15'::varchar::INTERVAL YEAR; +select '15'::bpchar(2)::INTERVAL YEAR; \ No newline at end of file From 5c2d39f3adbda4309bf46c9614accaf95da4b231 Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Tue, 18 Jul 2023 16:42:52 +0800 Subject: [PATCH 031/304] =?UTF-8?q?=E5=8F=8C=E9=9B=86=E7=BE=A4build=20chec?= =?UTF-8?q?k?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/pg_ctl/backup.cpp | 30 +-- src/bin/pg_ctl/pg_build.cpp | 11 +- src/bin/pg_ctl/pg_build.h | 1 - src/bin/pg_ctl/pg_ctl.cpp | 199 +++++++++++++++--- src/bin/pg_rewind/fetch.cpp | 1 + src/bin/pg_rewind/file_ops.cpp | 1 + src/bin/pg_rewind/parsexlog.cpp | 20 +- src/bin/pg_rewind/pg_rewind.cpp | 128 +++++++++++ src/bin/pg_rewind/pg_rewind.h | 1 + src/common/port/tool_common.cpp | 12 ++ .../storage/access/transam/xlogreader.cpp | 22 +- src/include/access/xlogreader.h | 2 +- src/include/replication/replicainternal.h | 3 +- src/include/tool_common.h | 13 +- 14 files changed, 369 insertions(+), 75 deletions(-) diff --git a/src/bin/pg_ctl/backup.cpp b/src/bin/pg_ctl/backup.cpp index b4c65ff5c..f86887fed 100755 --- a/src/bin/pg_ctl/backup.cpp +++ b/src/bin/pg_ctl/backup.cpp @@ -1014,9 +1014,9 @@ static bool ReceiveAndUnpackTarFile(PGconn* conn, PGresult* res, int rownum) } /* pg_control will be written into a specified postion of main stanby corresponding to */ - if (instance_config.dss.enable_dss && strcmp(filename, "+data/pg_control") == 0) { + if (ss_instance_config.dss.enable_dss && strcmp(filename, "+data/pg_control") == 0) { pg_log(PG_WARNING, _("file size %d. \n"), r); - int main_standby_id = instance_config.dss.instance_id; + int main_standby_id = ss_instance_config.dss.instance_id; off_t seekpos = (off_t)BLCKSZ * main_standby_id; fseek(file, seekpos, SEEK_SET); } @@ -1180,7 +1180,7 @@ static bool BaseBackup(const char* dirname, uint32 term) errno_t rc = EOK; int nRet = 0; struct stat st; - char *dssdir = instance_config.dss.vgdata; + char *dssdir = ss_instance_config.dss.vgname; pqsignal(SIGCHLD, BuildReaper); /* handle child termination */ /* concat file and path */ @@ -1227,7 +1227,7 @@ static bool BaseBackup(const char* dirname, uint32 term) delete_datadir(dirname); /* delete data/ and pg_tblspc/ in dss, but keep .config */ - if (instance_config.dss.enable_dss) { + if (ss_instance_config.dss.enable_dss) { delete_datadir(dssdir); } show_full_build_process("clear old target dir success"); @@ -1533,7 +1533,7 @@ static bool BaseBackup(const char* dirname, uint32 term) * in order to avoid sharing the same dssserver session, * we will not start logstreaming here */ - if (!instance_config.dss.enable_dss) { + if (!ss_instance_config.dss.enable_dss) { BeginGetXlogbyStream(xlogstart, timeline, sysidentifier, xlog_location, term, res); } @@ -1657,7 +1657,7 @@ static bool BaseBackup(const char* dirname, uint32 term) } #endif - if (instance_config.dss.enable_dss) { + if (ss_instance_config.dss.enable_dss) { BeginGetXlogbyStream(xlogstart, timeline, sysidentifier, xlog_location, term, res); } @@ -1749,7 +1749,7 @@ static bool BaseBackup(const char* dirname, uint32 term) /* fsync all data come from source */ if (!no_need_fsync) { show_full_build_process("starting fsync all files come from source."); - if (instance_config.dss.enable_dss) { + if (ss_instance_config.dss.enable_dss) { (void) fsync_pgdata(dssdir); } else { (void) fsync_pgdata(basedir); @@ -1762,7 +1762,7 @@ static bool BaseBackup(const char* dirname, uint32 term) if (g_is_obsmode) { backupDWFileSuccess = backup_dw_file(basedir); } else { - if (instance_config.dss.enable_dss) { + if (ss_instance_config.dss.enable_dss) { backupDWFileSuccess = ss_backup_dw_file(dssdir); } else { backupDWFileSuccess = backup_dw_file(dirname); @@ -1789,7 +1789,7 @@ static bool BaseBackup(const char* dirname, uint32 term) return false; } - if (instance_config.dss.enable_dss) { + if (ss_instance_config.dss.enable_dss) { nRet = snprintf_s(tblspcPath, MAXPGPATH, MAXPGPATH, "%s/pg_tblspc", dssdir); } else { nRet = snprintf_s(tblspcPath, MAXPGPATH, MAXPGPATH, "%s/pg_tblspc", dirname); @@ -2370,7 +2370,7 @@ static bool ss_backup_dw_file(const char* target_dir) /* Delete the dw file, if it exists. */ rc = snprintf_s(dw_path, PATH_MAX, PATH_MAX - 1, "%s/pg_doublewrite%d", target_dir, - instance_config.dss.instance_id); + ss_instance_config.dss.instance_id); securec_check_ss_c(rc, "\0", "\0"); /* check whether directory is exits or not, if not exit then mkdir it */ @@ -2506,10 +2506,10 @@ void get_xlog_location(char (&xlog_location)[MAXPGPATH]) struct stat stbuf; int nRet = 0; - if (instance_config.dss.enable_dss) { - char *dssdir = instance_config.dss.vgdata; + if (ss_instance_config.dss.enable_dss) { + char *dssdir = ss_instance_config.dss.vgname; nRet = snprintf_s(xlog_location, MAXPGPATH, MAXPGPATH - 1, "%s/pg_xlog%d", dssdir, - instance_config.dss.instance_id); + ss_instance_config.dss.instance_id); } else { nRet = snprintf_s(xlog_location, MAXPGPATH, MAXPGPATH - 1, "%s/pg_xlog", basedir); } @@ -2725,8 +2725,8 @@ bool RenameTblspcDir(char *dataDir) return true; } - if (instance_config.dss.enable_dss) { - char *dssdir = instance_config.dss.vgdata; + if (ss_instance_config.dss.enable_dss) { + char *dssdir = ss_instance_config.dss.vgname; rc = snprintf_s(tblspcParentPath, MAXPGPATH, MAXPGPATH - 1, "%s/%s", dssdir, "pg_tblspc"); } else { rc = snprintf_s(tblspcParentPath, MAXPGPATH, MAXPGPATH - 1, "%s/%s", dataDir, "pg_tblspc"); diff --git a/src/bin/pg_ctl/pg_build.cpp b/src/bin/pg_ctl/pg_build.cpp index 1a6805f4a..6966b672f 100755 --- a/src/bin/pg_ctl/pg_build.cpp +++ b/src/bin/pg_ctl/pg_build.cpp @@ -72,7 +72,6 @@ char g_repl_uuid[MAX_VALUE_LEN] = {0}; int g_replconn_idx = -1; int g_replication_type = -1; bool is_cross_region_build = false; -SSInstanceConfig instance_config; #define RT_WITH_DUMMY_STANDBY 0 #define RT_WITH_MULTI_STANDBY 1 @@ -610,7 +609,7 @@ void get_conninfo(const char* filename) } if (build_mode == CROSS_CLUSTER_FULL_BUILD || build_mode == CROSS_CLUSTER_INC_BUILD || - build_mode == CROSS_CLUSTER_STANDBY_FULL_BUILD) { + build_mode == CROSS_CLUSTER_STANDBY_FULL_BUILD || build_mode == BUILD_CHECK) { /* For shared storage cluster */ conninfo_para = config_para_cross_cluster_build; } else { @@ -1238,7 +1237,7 @@ int IsBeginWith(const char *str1, char *str2) bool SsIsSkipPath(const char* dirname, bool needskipall) { - if (!instance_config.dss.enable_dss) { + if (!ss_instance_config.dss.enable_dss) { return false; } @@ -1271,7 +1270,7 @@ bool SsIsSkipPath(const char* dirname, bool needskipall) char instanceId[MAX_INSTANCEID_LEN] = {0}; errno_t rc = EOK; rc = snprintf_s(instanceId, sizeof(instanceId), sizeof(instanceId) - 1, "%d", - instance_config.dss.instance_id); + ss_instance_config.dss.instance_id); securec_check_ss_c(rc, "\0", "\0"); /* not skip pg_xlog directory in file systerm */ if (strlen(dirname) > dirNameLen && strcmp(dirname + dirNameLen, instanceId) != 0) @@ -1575,7 +1574,7 @@ void delete_datadir(const char* dirname) */ if (strncmp(dirname, "+", 1) == 0 ) { nRet = snprintf_s(xlogpath, MAXPGPATH, sizeof(xlogpath) - 1, "%s/pg_xlog%d", dirname, - instance_config.dss.instance_id); + ss_instance_config.dss.instance_id); } else { nRet = snprintf_s(xlogpath, MAXPGPATH, sizeof(xlogpath) - 1, "%s/pg_xlog", dirname); } @@ -1788,7 +1787,7 @@ void fsync_pgdata(const char *pg_data) if (is_dss_file(pg_data)) { errorno = snprintf_s(pg_xlog, MAXPGPATH, MAXPGPATH - 1, "%s/pg_xlog%d", pg_data, - instance_config.dss.instance_id); + ss_instance_config.dss.instance_id); } else { errorno = snprintf_s(pg_xlog, MAXPGPATH, MAXPGPATH - 1, "%s/pg_xlog", pg_data); } diff --git a/src/bin/pg_ctl/pg_build.h b/src/bin/pg_ctl/pg_build.h index f45848cca..982c9f0ee 100755 --- a/src/bin/pg_ctl/pg_build.h +++ b/src/bin/pg_ctl/pg_build.h @@ -44,7 +44,6 @@ extern char conninfo_global[MAX_REPLNODE_NUM][MAX_VALUE_LEN]; extern int standby_recv_timeout; extern int standby_connect_timeout; /* 120 sec = default */ extern char gaussdb_state_file[MAXPGPATH]; -extern SSInstanceConfig instance_config; void delete_datadir(const char* dirname); diff --git a/src/bin/pg_ctl/pg_ctl.cpp b/src/bin/pg_ctl/pg_ctl.cpp index 55c63f6de..42ecb29f2 100755 --- a/src/bin/pg_ctl/pg_ctl.cpp +++ b/src/bin/pg_ctl/pg_ctl.cpp @@ -223,7 +223,7 @@ static char* new_role = "passive"; static char start_minority_file[MAXPGPATH]; static int get_instance_id(void); static int ss_get_primary_id(void); -static bool ss_read_dorado_config(void); +bool ss_read_config(void); static unsigned int vote_num = 0; static unsigned int xmode = 2; static char postport_lock_file[MAXPGPATH]; @@ -368,6 +368,7 @@ static void do_overwrite(void); static void do_full_restore(void); static void kill_proton_force(void); static void SigAlarmHandler(int arg); +static bool DoBuildCheck(uint32 term); int ExecuteCmd(const char* command, struct timeval timeout); static int find_guc_optval(const char** optlines, const char* optname, char* optval); @@ -4373,6 +4374,10 @@ static void do_build(uint32 term) else if (build_mode == COPY_SECURE_FILES_BUILD) { buildSuccess = DoCopySecureFileBuild(term); } + /* check need we build and can we do inc build */ + else if (build_mode == BUILD_CHECK) { + buildSuccess = DoBuildCheck(term); + } if (!buildSuccess) { exit(1); @@ -4805,6 +4810,116 @@ static bool GetRemoteNodeName() return true; } + +/* + * @@GaussDB@@ + * Brief : the check need to build + * Description : + * Notes : + */ +bool build_check_main(uint32 term) +{ + BuildErrorCode status = BUILD_SUCCESS; + PGresult* res = NULL; + errno_t errorno = EOK; + char* sysidentifier = NULL; + uint32 timeline; + char connstrSource[MAXPGPATH] = {0}; + g_inc_fail_reason = DEFAULT_REASON; + + CheckBuildParameter(); + check_nested_pgconf(); + + /* + * Save connection info from command line or openGauss file. + */ + get_conninfo(pg_conf_file); + + /* Find a available connection. */ + streamConn = check_and_conn(standby_connect_timeout, standby_recv_timeout, term); + if (streamConn == NULL) { + pg_log(PG_WARNING, _("could not connect to server.\n")); + g_inc_fail_reason = CONN_PRIMARY_FAIL; + return false; + } + + /* Concate connection str to primary host for performing rewind. */ + errorno = sprintf_s(connstrSource, + sizeof(connstrSource), + "host=%s port=%s dbname=postgres application_name=gs_rewind connect_timeout=5 rw_timeout=600", + (streamConn->pghost != NULL) ? streamConn->pghost : streamConn->pghostaddr, + streamConn->pgport); + securec_check_ss_c(errorno, "\0", "\0"); + + /* + * Run IDENTIFY_SYSTEM so we can get sys identifier and timeline. + */ + res = PQexec(streamConn, "IDENTIFY_SYSTEM"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) { + pg_log(PG_WARNING, _("could not identify system: %s"), PQerrorMessage(streamConn)); + PQfinish(streamConn); + streamConn = NULL; + PQclear(res); + return false; + } + if (PQntuples(res) != 1 || PQnfields(res) != 4) { + pg_log(PG_WARNING, _("could not identify system, got %d rows and %d fields\n"), PQntuples(res), PQnfields(res)); + PQfinish(streamConn); + streamConn = NULL; + PQclear(res); + return false; + } + sysidentifier = pg_strdup(PQgetvalue(res, 0, 0)); + timeline = atoi(PQgetvalue(res, 0, 1)); + PQclear(res); + + if (streamConn != NULL) { + PQfinish(streamConn); + streamConn = NULL; + } + + /* Pretend to be gs_rewind and perform rewind. */ + progname = "gs_rewind"; + status = do_build_check(pg_data, connstrSource, sysidentifier, timeline, term); + + libpqDisconnect(); + + if (sysidentifier != NULL) { + pg_free(sysidentifier); + sysidentifier = NULL; + } + + return (status == BUILD_SUCCESS); +} + +/* + * build_mode: + * check: check we are or not need to build + */ +static bool DoBuildCheck(uint32 term) +{ + bool buildSuccess = false; + char cwd[MAXPGPATH]; + + if (getcwd(cwd, MAXPGPATH) == NULL) { + pg_fatal(_("could not identify current directory: %s"), gs_strerror(errno)); + exit(1); + } + pg_log(PG_WARNING, _("current workdir is (%s).\n"), cwd); + + check_nested_pgconf(); + + replconn_num = get_replconn_number(pg_conf_file); + + buildSuccess = build_check_main(term); + if (!buildSuccess) { + pg_log(PG_WARNING, _("%s failed(%s), need to do full build\n"), BuildModeToString(build_mode), pg_data); + } else { + pg_log(PG_WARNING, _("%s completed(%s).\n"), BuildModeToString(build_mode), pg_data); + } + return buildSuccess; +} + /* * build_mode: * AUTO_BUILD: do gs_rewind first, after failed 3 times, do full @@ -4962,18 +5077,18 @@ static int find_guc_optval(const char** optlines, const char* optname, char* opt int ret; errno_t rc = EOK; - lineno = find_gucoption(optlines, (const char*)optname, NULL, NULL, &offset, &len); + lineno = find_gucoption(optlines, (const char*)optname, NULL, NULL, &offset, &len, '\''); if (lineno != INVALID_LINES_IDX) { - rc = strncpy_s(optval, MAX_VALUE_LEN, optlines[lineno] + offset + 1, (size_t)(Min(len - 1, MAX_VALUE_LEN) - 1)); + rc = strncpy_s(optval, MAX_VALUE_LEN, optlines[lineno] + offset, (size_t)(Min(len, MAX_VALUE_LEN))); securec_check_c(rc, "", ""); return lineno; } - ret = snprintf_s(def_optname, sizeof(def_optname), sizeof(def_optname) - 1, "#%s", optname); + ret = snprintf_s(def_optname, sizeof(def_optname), sizeof(def_optname) - 1, "#%s", optname, '\''); securec_check_ss_c(ret, "\0", "\0"); lineno = find_gucoption(optlines, (const char*)def_optname, NULL, NULL, &offset, &len); if (lineno != INVALID_LINES_IDX) { - rc = strncpy_s(optval, MAX_VALUE_LEN, optlines[lineno] + offset + 1, (size_t)(Min(len - 1, MAX_VALUE_LEN) - 1)); + rc = strncpy_s(optval, MAX_VALUE_LEN, optlines[lineno] + offset, (size_t)(Min(len, MAX_VALUE_LEN))); securec_check_c(rc, "", ""); return lineno; } @@ -5074,6 +5189,8 @@ const char *BuildModeToString(BuildMode mode) break; case COPY_SECURE_FILES_BUILD: return "copy secure files build"; + case BUILD_CHECK: + return "build check"; default: return "unkwon"; break; @@ -6358,6 +6475,8 @@ int main(int argc, char** argv) } else if (strcmp(optarg, "copy_upgrade_file") == 0) { build_mode = COPY_SECURE_FILES_BUILD; need_copy_upgrade_file = true; + } else if (strcmp(optarg, "check") == 0) { + build_mode = BUILD_CHECK; } break; } @@ -6692,7 +6811,7 @@ int main(int argc, char** argv) MIN_INSTANCEID, MAX_INSTANCEID ); goto Error; } - instance_config.dss.instance_id = atoi(optarg); + ss_instance_config.dss.instance_id = atoi(optarg); break; case 1: clear_backup_dir = true; @@ -6725,9 +6844,9 @@ int main(int argc, char** argv) FREE_AND_RESET(vgdata); FREE_AND_RESET(vgdata); parse_vgname_args(optarg); - instance_config.dss.vgname = xstrdup(vgname); - instance_config.dss.vgdata = xstrdup(vgdata); - instance_config.dss.vglog = xstrdup(vglog); + ss_instance_config.dss.vgname = xstrdup(vgname); + ss_instance_config.dss.vgdata = xstrdup(vgdata); + ss_instance_config.dss.vglog = xstrdup(vglog); break; } case 6:{ @@ -6737,11 +6856,11 @@ int main(int argc, char** argv) goto Error; } socketpath = xstrdup(optarg); - instance_config.dss.socketpath = xstrdup(optarg); + ss_instance_config.dss.socketpath = xstrdup(optarg); break; } case 7: - instance_config.dss.enable_dss = true; + ss_instance_config.dss.enable_dss = true; break; case 8:{ check_input_for_security(optarg); @@ -6918,21 +7037,25 @@ int main(int argc, char** argv) do_wait = false; } - if (instance_config.dss.enable_dss) { + enable_dss = ss_read_config(); + if (ss_instance_config.dss.enable_dss) { // dss device init - if (dss_device_init(instance_config.dss.socketpath, - instance_config.dss.enable_dss) != DSS_SUCCESS) { + if (dss_device_init(ss_instance_config.dss.socketpath, + ss_instance_config.dss.enable_dss) != DSS_SUCCESS) { pg_log(PG_WARNING, _("failed to init dss device\n")); goto Error; } /* Prepare some g_datadir parameters */ - g_datadir.instance_id = instance_config.dss.instance_id; + g_datadir.instance_id = ss_instance_config.dss.instance_id; - errno_t rc = strcpy_s(g_datadir.dss_data, strlen(instance_config.dss.vgdata) + 1, instance_config.dss.vgdata); + errno_t rc = strcpy_s(g_datadir.dss_data, strlen(ss_instance_config.dss.vgname) + 1, ss_instance_config.dss.vgname); securec_check_c(rc, "\0", "\0"); - rc = strcpy_s(g_datadir.dss_log, strlen(instance_config.dss.vglog) + 1, instance_config.dss.vglog); + if (ss_instance_config.dss.vglog == NULL) { + ss_instance_config.dss.vglog = ss_instance_config.dss.vgname; + } + rc = strcpy_s(g_datadir.dss_log, strlen(ss_instance_config.dss.vglog) + 1, ss_instance_config.dss.vglog); securec_check_c(rc, "\0", "\0"); /* The default of XLogSegmentSize was set 16M during configure, we reassign 1G to XLogSegmentSize @@ -6940,7 +7063,7 @@ int main(int argc, char** argv) XLogSegmentSize = DSS_XLOG_SEG_SIZE; } - initDataPathStruct(instance_config.dss.enable_dss); + initDataPathStruct(ss_instance_config.dss.enable_dss); SetConfigFilePath(); @@ -7057,6 +7180,11 @@ int main(int argc, char** argv) _("gs_ctl copy secure files from remote build ,datadir is %s,conn_str is \'%s\'\n"), pg_data, conn_str); + } else if (build_mode == BUILD_CHECK) { + pg_log(PG_PROGRESS, + _("gs_ctl build check ,datadir is %s,conn_str is \'%s\'\n"), + pg_data, + conn_str); } else { pg_log(PG_PROGRESS, _("gs_ctl incremental build ,datadir is %s,conn_str is \'%s\'\n"), @@ -7076,6 +7204,10 @@ int main(int argc, char** argv) pg_log(PG_PROGRESS, _("gs_ctl full backup to obs ,datadir is %s\n"), pg_data); + } else if (build_mode == BUILD_CHECK) { + pg_log(PG_PROGRESS, + _("gs_ctl build check ,datadir is %s\n"), + pg_data); } else { pg_log(PG_PROGRESS, _("gs_ctl incremental build ,datadir is %s\n"), @@ -7250,11 +7382,11 @@ static int get_instance_id(void) static int ss_get_primary_id(void) { - if (instance_config.dss.socketpath == NULL) { + if (ss_instance_config.dss.socketpath == NULL) { return -1; } - if (instance_config.dss.vgname == NULL) { + if (ss_instance_config.dss.vgname == NULL) { return -1; } @@ -7266,10 +7398,10 @@ static int ss_get_primary_id(void) err = memset_s(control_file_path, MAXPGPATH, 0, MAXPGPATH); securec_check_c(err, "\0", "\0"); - err = snprintf_s(control_file_path, MAXPGPATH, MAXPGPATH - 1, "%s/pg_control", instance_config.dss.vgname); + err = snprintf_s(control_file_path, MAXPGPATH, MAXPGPATH - 1, "%s/pg_control", ss_instance_config.dss.vgname); securec_check_ss_c(err, "\0", "\0"); - if (dss_device_init(instance_config.dss.socketpath, true) != DSS_SUCCESS) { + if (dss_device_init(ss_instance_config.dss.socketpath, true) != DSS_SUCCESS) { pg_log(PG_WARNING, _("failed to init dss device\n")); exit(1); } @@ -7307,12 +7439,13 @@ static int ss_get_primary_id(void) } /* -* read dorado config, if it is dorado standby cluster, -* we will get ss_dss_conn_path and ss_dss_vg_name. +* read ss config, return enable_dss +* we will get ss_enable_dss, ss_dss_conn_path and ss_dss_vg_name. */ -static bool ss_read_dorado_config(void) +bool ss_read_config(void) { char config_file[MAXPGPATH] = {0}; + char enable_dss[MAXPGPATH] = {0}; char** optlines = NULL; int ret = EOK; @@ -7320,19 +7453,19 @@ static bool ss_read_dorado_config(void) securec_check_ss_c(ret, "\0", "\0"); config_file[MAXPGPATH - 1] = '\0'; optlines = readfile(config_file); - char cluster_run_mode[MAXPGPATH] = {0}; - (void)find_guc_optval((const char**)optlines, "cluster_run_mode", cluster_run_mode); + (void)find_guc_optval((const char**)optlines, "ss_enable_dss", enable_dss); - /* this is not dorado cluster_standby, wo do not need to do anythiny else */ - if(strncmp(cluster_run_mode, "cluster_standby", sizeof("cluster_standby")) != 0) { + /* this is not enable_dss, wo do not need to do anythiny else */ + if(strncmp(enable_dss, "on", sizeof("on")) != 0) { return false; } - instance_config.dss.socketpath = (char*)malloc(sizeof(char) * MAXPGPATH); - instance_config.dss.vgname = (char*)malloc(sizeof(char) * MAXPGPATH); - (void)find_guc_optval((const char**)optlines, "ss_dss_conn_path", instance_config.dss.socketpath); - (void)find_guc_optval((const char**)optlines, "ss_dss_vg_name", (char*)instance_config.dss.vgname); + ss_instance_config.dss.enable_dss = true; + ss_instance_config.dss.socketpath = (char*)malloc(sizeof(char) * MAXPGPATH); + ss_instance_config.dss.vgname = (char*)malloc(sizeof(char) * MAXPGPATH); + (void)find_guc_optval((const char**)optlines, "ss_dss_conn_path", ss_instance_config.dss.socketpath); + (void)find_guc_optval((const char**)optlines, "ss_dss_vg_name", ss_instance_config.dss.vgname); freefile(optlines); optlines = NULL; return true; diff --git a/src/bin/pg_rewind/fetch.cpp b/src/bin/pg_rewind/fetch.cpp index b52fc8f0e..03ad5b58f 100755 --- a/src/bin/pg_rewind/fetch.cpp +++ b/src/bin/pg_rewind/fetch.cpp @@ -26,6 +26,7 @@ #include "catalog/catalog.h" #include "PageCompression.h" #include "catalog/pg_type.h" +#include "storage/file/fio_device.h" PGconn* conn = NULL; char source_slot_name[NAMEDATALEN] = {0}; diff --git a/src/bin/pg_rewind/file_ops.cpp b/src/bin/pg_rewind/file_ops.cpp index 2ef066fa9..3ea3ea6cd 100644 --- a/src/bin/pg_rewind/file_ops.cpp +++ b/src/bin/pg_rewind/file_ops.cpp @@ -26,6 +26,7 @@ #include "common/fe_memutils.h" #include "common/build_query/build_query.h" #include "replication/replicainternal.h" +#include "storage/file/fio_device.h" #include #define BLOCKSIZE (8 * 1024) diff --git a/src/bin/pg_rewind/parsexlog.cpp b/src/bin/pg_rewind/parsexlog.cpp index e9ead584d..3c7ca5681 100644 --- a/src/bin/pg_rewind/parsexlog.cpp +++ b/src/bin/pg_rewind/parsexlog.cpp @@ -138,10 +138,12 @@ BuildErrorCode findCommonCheckpoint(const char* datadir, TimeLineID tli, XLogRec #ifdef HAVE_INT64_TIMESTAMP #define TIME_COUNT 60000000 #else - #define TIME_COUNT 60 + #define TIME_COUNT 300 #endif XLogRecPtr max_lsn; char returnmsg[MAX_ERR_MSG_LENTH] = {0}; + char dssdirdata[MAXPGPATH] = {0}; + char* dssdir = dssdirdata; pg_crc32 maxLsnCrc = 0; XLogRecord* record = NULL; XLogRecPtr searchptr; @@ -154,10 +156,17 @@ BuildErrorCode findCommonCheckpoint(const char* datadir, TimeLineID tli, XLogRec TimestampTz start_time; TimestampTz current_time; + if (ss_instance_config.dss.enable_dss) { + ret = snprintf_s(dssdirdata, MAXPGPATH, MAXPGPATH - 1, "%s/%s%d", ss_instance_config.dss.vgname, XLOGDIR, ss_instance_config.dss.instance_id); + securec_check_ss_c(ret, "", ""); + } else { + dssdir = NULL; + } + /* * local max lsn must be exists, or change to full build. */ - max_lsn = FindMaxLSN(datadir_target, returnmsg, XLOG_READER_MAX_MSGLENTH, &maxLsnCrc); + max_lsn = FindMaxLSN(datadir_target, returnmsg, XLOG_READER_MAX_MSGLENTH, &maxLsnCrc, NULL, NULL, dssdir); if (XLogRecPtrIsInvalid(max_lsn)) { pg_fatal("find max lsn fail, errmsg:%s\n", returnmsg); return BUILD_FATAL; @@ -177,20 +186,21 @@ BuildErrorCode findCommonCheckpoint(const char* datadir, TimeLineID tli, XLogRec securec_check_ss_c(ret, "\0", "\0"); get_conninfo(pg_conf_file); + searchptr = max_lsn; start_time = localGetCurrentTimestamp(); current_time = start_time; while (!XLogRecPtrIsInvalid(searchptr)) { if (current_time - start_time >= TIME_COUNT) { pg_log(PG_FATAL, - "try 60s, could not find any common checkpoint, change to full build\n"); + "try 300s, could not find any common checkpoint, need to do full build\n"); XLogReaderFree(xlogreader); CloseXlogFile(); return BUILD_FATAL; } uint8 info; - record = XLogReadRecord(xlogreader, searchptr, &errormsg); + record = XLogReadRecord(xlogreader, searchptr, &errormsg, true, dssdir); if (record == NULL) { if (errormsg != NULL) { pg_fatal("could not find previous WAL record at %X/%X: %s\n", @@ -245,7 +255,7 @@ BuildErrorCode findCommonCheckpoint(const char* datadir, TimeLineID tli, XLogRec PG_CHECKBUILD_AND_RETURN(); /* no common checkpoint between target and source, need full build */ if (XLogRecPtrIsInvalid(searchptr)) { - pg_log(PG_FATAL, "could not find any common checkpoint, change to full build\n"); + pg_log(PG_FATAL, "could not find any common checkpoint, must to do full build\n"); return BUILD_FATAL; } return BUILD_SUCCESS; diff --git a/src/bin/pg_rewind/pg_rewind.cpp b/src/bin/pg_rewind/pg_rewind.cpp index da0e1f953..3ee3318ca 100755 --- a/src/bin/pg_rewind/pg_rewind.cpp +++ b/src/bin/pg_rewind/pg_rewind.cpp @@ -22,6 +22,7 @@ #include "fetch.h" #include "file_ops.h" #include "logging.h" +#include "storage/file/fio_device.h" #include "access/xlog_internal.h" #include "catalog/catversion.h" @@ -931,3 +932,130 @@ static BuildErrorCode TruncateAndRemoveXLog(XLogRecPtr endPtr, uint32 timeLine) return BUILD_SUCCESS; } +BuildErrorCode do_build_check(const char* pgdata, const char* connstr, char* sysidentifier, uint32 timeline, uint32 term) +{ + TimeLineID lastcommontli; + XLogRecPtr chkptrec = InvalidXLogRecPtr; + TimeLineID chkpttli; + XLogRecPtr chkptredo = InvalidXLogRecPtr; + size_t size = 0; + char* buffer = NULL; + XLogRecPtr startrec; + errno_t errorno = EOK; + BuildErrorCode rv = BUILD_SUCCESS; + + datadir_target = pg_strdup(pgdata); + if (connstr_source == NULL) { + connstr_source = pg_strdup(connstr); + } + + if (connstr_source == NULL) { + pg_log(PG_WARNING, "%s: no source specified (--source-server)\n", progname); + pg_log(PG_WARNING, "Try \"%s --help\" for more information.\n", progname); + return BUILD_ERROR; + } + + if (datadir_target == NULL) { + pg_log(PG_WARNING, "%s: no target data directory specified (--target-pgdata)\n", progname); + pg_log(PG_WARNING, "Try \"%s --help\" for more information.\n", progname); + return BUILD_ERROR; + } + + if (term > PG_UINT32_MAX) { + pg_log(PG_PROGRESS, "%s: unexpected term specified\n", progname); + pg_log(PG_PROGRESS, "Try \"%s --help\" for more information.\n", progname); + return BUILD_ERROR; + } + + /* + * Don't allow pg_rewind to be run as root, to avoid overwriting the + * ownership of files in the data directory. We need only check for root + * -- any other user won't have sufficient permissions to modify files in + * the data directory. + */ + if (geteuid() == 0) { + pg_log(PG_PROGRESS, "cannot be executed by \"root\"\n"); + pg_log(PG_PROGRESS, "You must run %s as the PostgreSQL superuser.\n", progname); + exit(1); + } + + /* Can't start new building until restore process success. */ + if (is_in_restore_process(datadir_target)) { + pg_log(PG_PROGRESS, + "%s: last restore process hasn't completed, " + "can't start new building.\n", + progname); + return BUILD_ERROR; + } + + /* Connect to remote server */ + rv = libpqConnect(connstr_source); + PG_CHECKRETURN_AND_RETURN(rv); + rv = libpqGetParameters(); + PG_CHECKRETURN_AND_RETURN(rv); + pg_log(PG_PROGRESS, "connect to primary success\n"); + + /* + * Ok, we have all the options and we're ready to start. Read in all the + * information we need from both clusters. + */ + buffer = slurpFile(ss_instance_config.dss.vgname, "pg_control", &size); + PG_CHECKBUILD_AND_RETURN(); + digestControlFile(&ControlFile_target, (const char*)buffer); + pg_free(buffer); + buffer = NULL; + PG_CHECKBUILD_AND_RETURN(); + + pg_log(PG_PROGRESS, + "find last checkpoint at %X/%X and checkpoint redo at %X/%X from target control file\n", + (uint32)(ControlFile_target.checkPoint >> 32), + (uint32)(ControlFile_target.checkPoint), + (uint32)(ControlFile_target.checkPointCopy.redo >> 32), + (uint32)(ControlFile_target.checkPointCopy.redo)); + + buffer = fetchFile("+data/pg_control", &size); + PG_CHECKBUILD_AND_RETURN(); + digestControlFile(&ControlFile_source, buffer); + pg_free(buffer); + buffer = NULL; + PG_CHECKBUILD_AND_RETURN(); + pg_log(PG_PROGRESS, "get primary pg_control success\n"); + + /* Check if rewind can be performed */ + rv = sanityChecks(); + PG_CHECKRETURN_AND_RETURN(rv); + pg_log(PG_PROGRESS, "sanityChecks success\n"); + + lastcommontli = ControlFile_target.checkPointCopy.ThisTimeLineID; + + pg_log(PG_PROGRESS, + "find last checkpoint at %X/%X and checkpoint redo at %X/%X from source control file\n", + (uint32)(ControlFile_source.checkPoint >> 32), + (uint32)(ControlFile_source.checkPoint), + (uint32)(ControlFile_source.checkPointCopy.redo >> 32), + (uint32)(ControlFile_source.checkPointCopy.redo)); + + /* Find the common checkpoint locaiton */ + startrec = ControlFile_source.checkPoint <= ControlFile_target.checkPoint ? + ControlFile_source.checkPoint : ControlFile_target.checkPoint; + rv = findCommonCheckpoint(datadir_target, lastcommontli, startrec, &chkptrec, &chkpttli, &chkptredo, term); + PG_CHECKRETURN_AND_RETURN(rv); + pg_log(PG_PROGRESS, "find diverge point success\n"); + + if (chkptrec == ControlFile_target.checkPoint) { + pg_log(PG_PROGRESS, "do not need to build\n"); + } else { + pg_log(PG_PROGRESS, "need to do incremental build\n"); + } + /* Disconnect from remote server */ + if (connstr_source != NULL) { + libpqDisconnect(); + } + + if (datadir_target != NULL) { + free(datadir_target); + datadir_target = NULL; + } + + return BUILD_SUCCESS; +} diff --git a/src/bin/pg_rewind/pg_rewind.h b/src/bin/pg_rewind/pg_rewind.h index 546540832..2d505808b 100644 --- a/src/bin/pg_rewind/pg_rewind.h +++ b/src/bin/pg_rewind/pg_rewind.h @@ -67,6 +67,7 @@ extern BuildErrorCode targetFilemapProcess(void); void recordReadTest(const char* datadir, XLogRecPtr ptr, TimeLineID tli); void openDebugLog(void); bool FindConfirmedLSN(const char* dataDir, XLogRecPtr *confirmedLsn); +BuildErrorCode do_build_check(const char* pgdata, const char* connstr, char* sysidentifier, uint32 timeline, uint32 term); BuildErrorCode CheckConfirmedLSNOnTarget(const char *datadir, TimeLineID tli, XLogRecPtr ckptRedo, XLogRecPtr confirmedLSN, uint32 term); bool CheckIfEanbedSaveSlots(); diff --git a/src/common/port/tool_common.cpp b/src/common/port/tool_common.cpp index aa424b437..81cb44d10 100644 --- a/src/common/port/tool_common.cpp +++ b/src/common/port/tool_common.cpp @@ -25,6 +25,18 @@ #include "securec_check.h" #include "tool_common.h" +SSInstanceConfig ss_instance_config = { + .dss = { + .enable_dss = false, + .instance_id = -1, + .primaryInstId = -1, + .vgname = NULL, + .vglog = NULL, + .vgdata = NULL, + .socketpath = NULL, + }, +}; + datadir_t g_datadir; /* need init when used in first time */ static void initFileDataPathStruct(datadir_t *dataDir); diff --git a/src/gausskernel/storage/access/transam/xlogreader.cpp b/src/gausskernel/storage/access/transam/xlogreader.cpp index 73729d16e..8b88abfe4 100644 --- a/src/gausskernel/storage/access/transam/xlogreader.cpp +++ b/src/gausskernel/storage/access/transam/xlogreader.cpp @@ -33,6 +33,7 @@ #include "utils/memutils.h" #include "utils/elog.h" #include "ddes/dms/ss_dms_recovery.h" +#include "storage/file/fio_device.h" typedef struct XLogPageReadPrivate { const char *datadir; @@ -1268,7 +1269,11 @@ int SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, in securec_check_ss_c(ss_c, "", ""); #endif - ss_c = snprintf_s(xlogfpath, MAXPGPATH, MAXPGPATH - 1, "%s/" XLOGDIR "/%s", readprivate->datadir, xlogfname); + if (xlog_path != NULL) { + ss_c = snprintf_s(xlogfpath, MAXPGPATH, MAXPGPATH - 1, "%s/%s", xlog_path, xlogfname); + } else { + ss_c = snprintf_s(xlogfpath, MAXPGPATH, MAXPGPATH - 1, "%s/" XLOGDIR "/%s", readprivate->datadir, xlogfname); + } #ifndef FRONTEND securec_check_ss(ss_c, "", ""); #else @@ -1313,8 +1318,8 @@ int SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, in return XLOG_BLCKSZ; } -XLogRecPtr FindMaxLSN(char *workingPath, char *returnMsg, int msgLen, pg_crc32 *maxLsnCrc, uint32 *maxLsnLen, - TimeLineID *returnTli) +XLogRecPtr FindMaxLSN(char *workingPath, char *returnMsg, int msgLen, pg_crc32 *maxLsnCrc, + uint32 *maxLsnLen, TimeLineID *returnTli, char* xlog_path) { DIR *xlogDir = NULL; struct dirent *dirEnt = NULL; @@ -1336,7 +1341,12 @@ XLogRecPtr FindMaxLSN(char *workingPath, char *returnMsg, int msgLen, pg_crc32 * uint32 xlogReadLogSeg = -1; errno_t rc = EOK; - rc = snprintf_s(xlogDirStr, MAXPGPATH, MAXPGPATH - 1, "%s/%s", workingPath, XLOGDIR); + if (xlog_path != NULL) { + rc = snprintf_s(xlogDirStr, MAXPGPATH, MAXPGPATH - 1, "%s", xlog_path); + } else { + rc = snprintf_s(xlogDirStr, MAXPGPATH, MAXPGPATH - 1, "%s/%s", workingPath, XLOGDIR); + } + #ifndef FRONTEND securec_check_ss(rc, "", ""); #else @@ -1410,7 +1420,7 @@ XLogRecPtr FindMaxLSN(char *workingPath, char *returnMsg, int msgLen, pg_crc32 * startLsn = (xlogReadLogSeg * XLogSegSize) + ((XLogRecPtr)xlogReadLogid * XLogSegmentsPerXLogId * XLogSegSize); while (!XLogRecPtrIsInvalid(startLsn)) { /* find the first valid record from the bigger xlogrecord. then break */ - curLsn = XLogFindNextRecord(xlogReader, startLsn); + curLsn = XLogFindNextRecord(xlogReader, startLsn, NULL, xlogDirStr); if (XLogRecPtrIsInvalid(curLsn)) { if (xlogreadfd > 0) { close(xlogreadfd); @@ -1446,7 +1456,7 @@ XLogRecPtr FindMaxLSN(char *workingPath, char *returnMsg, int msgLen, pg_crc32 * /* find the max lsn. */ do { - record = XLogReadRecord(xlogReader, curLsn, &errorMsg); + record = XLogReadRecord(xlogReader, curLsn, &errorMsg, true, xlogDirStr); if (record == NULL) { break; } diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index cf461f5e7..df3309928 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -57,7 +57,7 @@ extern void XLogReaderInvalReadState(XLogReaderState* state); extern XLogRecPtr XLogFindNextRecord(XLogReaderState* state, XLogRecPtr RecPtr, XLogRecPtr *endPtr = NULL, char* xlog_path = NULL); extern XLogRecPtr FindMaxLSN(char* workingpath, char* returnmsg, int msg_len, pg_crc32* maxLsnCrc, - uint32 *maxLsnLen = NULL, TimeLineID *returnTli = NULL); + uint32 *maxLsnLen = NULL, TimeLineID *returnTli = NULL, char* xlog_path = NULL); extern XLogRecPtr FindMinLSN(char *workingPath, char *returnMsg, int msgLen, pg_crc32 *minLsnCrc); extern void CloseXlogFile(void); extern int SimpleXLogPageRead(XLogReaderState* xlogreader, XLogRecPtr targetPagePtr, int reqLen, diff --git a/src/include/replication/replicainternal.h b/src/include/replication/replicainternal.h index ea1c85d1f..b582b44e6 100755 --- a/src/include/replication/replicainternal.h +++ b/src/include/replication/replicainternal.h @@ -104,7 +104,8 @@ typedef enum { COPY_SECURE_FILES_BUILD, CROSS_CLUSTER_FULL_BUILD, CROSS_CLUSTER_INC_BUILD, - CROSS_CLUSTER_STANDBY_FULL_BUILD + CROSS_CLUSTER_STANDBY_FULL_BUILD, + BUILD_CHECK } BuildMode; typedef struct buildstate { diff --git a/src/include/tool_common.h b/src/include/tool_common.h index bf75bc0e6..e126f0a90 100644 --- a/src/include/tool_common.h +++ b/src/include/tool_common.h @@ -105,23 +105,22 @@ typedef struct st_datadir_t { dw_subdatadir_t dwDir; } datadir_t; -typedef struct DssOptions -{ +/* DSS conntct parameters */ +typedef struct DssOptions { bool enable_dss; int instance_id; - const char *vgname; + int primaryInstId; + char *vgname; char *vglog; char *vgdata; char *socketpath; - int primaryInstId; } DssOptions; -typedef struct SSInstanceConfig -{ - /* DSS conntct parameters */ +typedef struct SSInstanceConfig { DssOptions dss; } SSInstanceConfig; +extern SSInstanceConfig ss_instance_config; extern datadir_t g_datadir; void initDataPathStruct(bool enable_dss); From 4061eab2df47b6294072f7b6da04635dbe00c0e4 Mon Sep 17 00:00:00 2001 From: zhang_xubo <2578876417@qq.com> Date: Tue, 18 Jul 2023 19:49:35 +0800 Subject: [PATCH 032/304] =?UTF-8?q?do=5Fstop=E5=88=A4=E6=96=AD=E8=BF=9B?= =?UTF-8?q?=E7=A8=8B=E6=98=AF=E5=90=A6=E5=AD=98=E5=9C=A8=E9=87=8D=E6=96=B0?= =?UTF-8?q?=E8=8E=B7=E5=8F=96=E4=B8=8Bpid?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/pg_ctl/pg_ctl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bin/pg_ctl/pg_ctl.cpp b/src/bin/pg_ctl/pg_ctl.cpp index 55c63f6de..4c2a77e24 100755 --- a/src/bin/pg_ctl/pg_ctl.cpp +++ b/src/bin/pg_ctl/pg_ctl.cpp @@ -1933,7 +1933,7 @@ static void do_stop(bool force) break; } - if (pid != 0) { /* pid file still exists */ + if ((pid = get_pgpid()) != 0) { /* pid file still exists */ if (force || (shutdown_mode == IMMEDIATE_MODE)) { kill_proton_force(); From 13d342a8a6c17a69deb09ea9433a1d539e924d82 Mon Sep 17 00:00:00 2001 From: chenxiaobin19 <1025221611@qq.com> Date: Tue, 18 Jul 2023 20:11:49 +0800 Subject: [PATCH 033/304] fix ignore_standby_lsn_window/ignore_feedback_xmin_window typo --- src/bin/gs_guc/cluster_guc.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bin/gs_guc/cluster_guc.conf b/src/bin/gs_guc/cluster_guc.conf index 67ec541d0..986e134d3 100755 --- a/src/bin/gs_guc/cluster_guc.conf +++ b/src/bin/gs_guc/cluster_guc.conf @@ -748,8 +748,8 @@ dolphin_server_port|int|1024,65535|NULL|NULL| enable_dolphin_proto|bool|0,0|NULL|NULL| enable_remote_excute|bool|0,0|NULL|NULL| light_comm|bool|0,0|NULL|NULL| -ignore_standby_lsn_window|int|0,2147483647|s|NULL| -ignore_feedback_xmin_window|int|0,2147483647|s|NULL| +ignore_standby_lsn_window|int|0,2147483647|ms|NULL| +ignore_feedback_xmin_window|int|0,2147483647|ms|NULL| [cmserver] log_dir|string|0,0|NULL|NULL| log_file_size|int|0,2047|MB|NULL| From 5e8380f9811c824c263f7bda34d7f62fcb9b5795 Mon Sep 17 00:00:00 2001 From: jiangyan <18091841830@163.com> Date: Wed, 19 Jul 2023 15:19:35 +0800 Subject: [PATCH 034/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dalter=20system=20set?= =?UTF-8?q?=E5=86=85=E5=AD=98=E6=B3=84=E6=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/storage/replication/syncrep.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/gausskernel/storage/replication/syncrep.cpp b/src/gausskernel/storage/replication/syncrep.cpp index 15babb36c..356fe9ea8 100755 --- a/src/gausskernel/storage/replication/syncrep.cpp +++ b/src/gausskernel/storage/replication/syncrep.cpp @@ -2336,6 +2336,9 @@ void assign_synchronous_standby_names(const char *newval, void *extra) i++; } + list_free_deep(tcxt->SyncRepConfig); + tcxt->SyncRepConfig = NIL; + (void)MemoryContextSwitchTo(old_context); } From 58f8b19f9a23cac69dbd5d4064330accb5c02517 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cross-=E7=BD=97?= <1165977584@qq.com> Date: Wed, 19 Jul 2023 08:45:05 +0000 Subject: [PATCH 035/304] =?UTF-8?q?update=20src/gausskernel/process/postma?= =?UTF-8?q?ster/pgarch.cpp.=20=E4=BF=AE=E5=A4=8Dsnprintf=5Fs=E4=BD=BF?= =?UTF-8?q?=E7=94=A8=E4=B8=8D=E8=A7=84=E8=8C=83=E5=AF=BC=E8=87=B4=E5=BD=92?= =?UTF-8?q?=E6=A1=A3=E8=B7=AF=E5=BE=84=E9=94=99=E8=AF=AF=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Cross-罗 <1165977584@qq.com> --- src/gausskernel/process/postmaster/pgarch.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/gausskernel/process/postmaster/pgarch.cpp b/src/gausskernel/process/postmaster/pgarch.cpp index 4322ae26a..64d2b6203 100755 --- a/src/gausskernel/process/postmaster/pgarch.cpp +++ b/src/gausskernel/process/postmaster/pgarch.cpp @@ -611,6 +611,7 @@ static bool PgarchArchiveXlogToDest(const char* xlog) int fdDest = -1; char srcPath[PATH_MAX + 1] = {0}; char destPath[PATH_MAX + 1] = {0}; + char archPath[PATH_MAX + 1] = {0}; char activitymsg[MAXFNAMELEN + 16]; long int fileBytes = 0; int rc = 0; @@ -631,17 +632,17 @@ static bool PgarchArchiveXlogToDest(const char* xlog) if (retVal == NULL) { ereport(FATAL, (errmsg_internal("realpath dest %s failed:%m\n", u_sess->attr.attr_storage.XLogArchiveDest))); } - rc = snprintf_s(destPath, PATH_MAX, PATH_MAX - 1, "%s/%s", destPath, xlog); + rc = snprintf_s(archPath, PATH_MAX, PATH_MAX - 1, "%s/%s", destPath, xlog); securec_check_ss(rc, "\0", "\0"); if ((fdSrc = open(srcPath, O_RDONLY)) >= 0) { - if ((fdDest = open(destPath, O_WRONLY | O_CREAT, S_IRUSR | S_IWUSR)) >= 0) { + if ((fdDest = open(archPath, O_WRONLY | O_CREAT, S_IRUSR | S_IWUSR)) >= 0) { char pbuff[ARCHIVE_BUF_SIZE] = {0}; while ((fileBytes = read(fdSrc, pbuff, sizeof(pbuff))) > 0) { if (write(fdDest, pbuff, fileBytes) != fileBytes) { close(fdSrc); - ereport(FATAL, (errmsg_internal("could not write file\"%s\":%m\n", destPath))); + ereport(FATAL, (errmsg_internal("could not write file\"%s\":%m\n", archPath))); } (void)memset_s(pbuff, sizeof(pbuff), 0, sizeof(pbuff)); } @@ -663,7 +664,7 @@ static bool PgarchArchiveXlogToDest(const char* xlog) return true; } else { close(fdSrc); - ereport(FATAL, (errmsg_internal("could not open archive dest file \"%s\":%m\n", destPath))); + ereport(FATAL, (errmsg_internal("could not open archive dest file \"%s\":%m\n", archPath))); } } else { ereport(FATAL, (errmsg_internal("could not open archive src file \"%s\":%m\n", srcPath))); From ba860581ca90a979f6ce24d101191c2e1d559d36 Mon Sep 17 00:00:00 2001 From: congzhou2603 Date: Thu, 6 Jul 2023 15:03:53 +0800 Subject: [PATCH 036/304] =?UTF-8?q?=E3=80=90feature=E3=80=91=E5=AD=90?= =?UTF-8?q?=E4=BA=8B=E5=8A=A1=E5=9B=9E=E6=BB=9A=E6=97=B6=E9=94=81=E6=8A=A2?= =?UTF-8?q?=E5=8D=A0=E9=97=AE=E9=A2=98=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../storage/access/transam/xact.cpp | 20 ++- src/gausskernel/storage/ipc/procarray.cpp | 150 ++++++++++++------ src/gausskernel/storage/lmgr/proc.cpp | 3 + src/include/storage/proc.h | 12 ++ src/include/storage/procarray.h | 6 +- 5 files changed, 141 insertions(+), 50 deletions(-) diff --git a/src/gausskernel/storage/access/transam/xact.cpp b/src/gausskernel/storage/access/transam/xact.cpp index 57bcd4ea5..d4b47eaa0 100755 --- a/src/gausskernel/storage/access/transam/xact.cpp +++ b/src/gausskernel/storage/access/transam/xact.cpp @@ -2177,8 +2177,24 @@ static TransactionId RecordTransactionAbort(bool isSubXact) * subxacts, because we already have the child XID array at hand. For * main xacts, the equivalent happens just after this function returns. */ - if (isSubXact) - XidCacheRemoveRunningXids(xid, nchildren, children, latestXid); + if (isSubXact) { + if (LWLockConditionalAcquire(ProcArrayLock, LW_EXCLUSIVE)) { + t_thrd.proc->procArrayGroupMemberXid = xid; + t_thrd.proc->procArrayGroupSubXactNXids = nchildren; + t_thrd.proc->procArrayGroupSubXactXids = children; + t_thrd.proc->procArrayGroupSubXactLatestXid = latestXid; + XidCacheRemoveRunningXids(t_thrd.proc, t_thrd.pgxact); + + /* clear the group member cache after XidCacheRemoveRunningXids*/ + t_thrd.proc->procArrayGroupMemberXid = InvalidTransactionId; + t_thrd.proc->procArrayGroupSubXactNXids = 0; + t_thrd.proc->procArrayGroupSubXactXids = NULL; + t_thrd.proc->procArrayGroupSubXactLatestXid = InvalidTransactionId; + LWLockRelease(ProcArrayLock); + } else { + ProcArrayGroupClearXid(true, t_thrd.proc, InvalidTransactionId, xid, nchildren, children, latestXid); + } + } /* Reset XactLastRecEnd until the next transaction writes something */ if (!isSubXact) diff --git a/src/gausskernel/storage/ipc/procarray.cpp b/src/gausskernel/storage/ipc/procarray.cpp index c8b43dd59..ab7216d0a 100755 --- a/src/gausskernel/storage/ipc/procarray.cpp +++ b/src/gausskernel/storage/ipc/procarray.cpp @@ -185,7 +185,10 @@ static TransactionId GetMultiSnapshotOldestXmin(); static inline void ProcArrayEndTransactionInternal(PGPROC* proc, PGXACT* pgxact, TransactionId latestXid, TransactionId* xid, uint32* nsubxids); -static void ProcArrayGroupClearXid(PGPROC* proc, TransactionId latestXid); +void XidCacheRemoveRunningXids(PGPROC* proc, PGXACT* pgxact); +void ProcArrayGroupClearXid(bool isSubTransaction, PGPROC* proc, TransactionId latestXid, + TransactionId subTranactionXid, int nSubTransactionXids, + TransactionId* subTransactionXids, TransactionId subTransactionLatestXid); extern bool StreamTopConsumerAmI(); @@ -522,7 +525,7 @@ void ProcArrayEndTransaction(PGPROC* proc, TransactionId latestXid, bool isCommi CalculateLocalLatestSnapshot(false); LWLockRelease(ProcArrayLock); } else - ProcArrayGroupClearXid(proc, latestXid); + ProcArrayGroupClearXid(false, proc, latestXid, InvalidTransactionId, 0, NULL, InvalidTransactionId); } else { /* * If we have no XID, we don't need to lock, since we won't affect @@ -603,6 +606,35 @@ static inline void ProcArrayEndTransactionInternal(PGPROC* proc, PGXACT* pgxact, ResetProcXidCache(proc, true); } +static inline void ProcInsertIntoGroup(PGPROC* proc, uint32* nextidx) { + while (true) { + *nextidx = pg_atomic_read_u32(&g_instance.proc_base->procArrayGroupFirst); + pg_atomic_write_u32(&proc->procArrayGroupNext, *nextidx); + + if (pg_atomic_compare_exchange_u32( + &g_instance.proc_base->procArrayGroupFirst, nextidx, (uint32)proc->pgprocno)) + break; + } + +} + +static inline void ClearProcArrayGroupCache(PGPROC* proc) { + proc->procArrayGroupMember = false; + proc->procArrayGroupMemberXid = InvalidTransactionId; + proc->procArrayGroupSubXactNXids = 0; + proc->procArrayGroupSubXactXids = NULL; + proc->procArrayGroupSubXactLatestXid = InvalidTransactionId; +} + +static inline void SetProcArrayGroupCache(PGPROC* proc, TransactionId xid, int nxids, + TransactionId* xids, TransactionId latestXid) +{ + proc->procArrayGroupMemberXid = xid; + proc->procArrayGroupSubXactNXids = nxids; + proc->procArrayGroupSubXactXids = xids; + proc->procArrayGroupSubXactLatestXid = latestXid; +} + /* * ProcArrayGroupClearXid -- group XID clearing * @@ -610,34 +642,36 @@ static inline void ProcArrayEndTransactionInternal(PGPROC* proc, PGXACT* pgxact, * commit time, add ourselves to a list of processes that need their XIDs * cleared. The first process to add itself to the list will acquire * ProcArrayLock in exclusive mode and perform ProcArrayEndTransactionInternal - * on behalf of all group members. This avoids a great deal of contention + * for transaction group members and XidCacheRemoveRunningXids + * for subtransaction group members. This avoids a great deal of contention * around ProcArrayLock when many processes are trying to commit at once, * since the lock need not be repeatedly handed off from one committing * process to the next. */ -static void ProcArrayGroupClearXid(PGPROC* proc, TransactionId latestXid) +void ProcArrayGroupClearXid(bool isSubTransaction, PGPROC* proc, + TransactionId latestXid, TransactionId subTranactionXid, + int nSubTransactionXids, TransactionId* subTransactionXids, + TransactionId subTransactionLatestXid) { uint32 nextidx; uint32 wakeidx; TransactionId xid[PROCARRAY_MAXPROCS]; uint32 nsubxids[PROCARRAY_MAXPROCS]; uint32 index = 0; - uint32 commitcsn[PROCARRAY_MAXPROCS]; - CommitSeqNo maxcsn = 0; + bool groupMemberHasTransaction = false; /* We should definitely have an XID to clear. */ /* Add ourselves to the list of processes needing a group XID clear. */ proc->procArrayGroupMember = true; - proc->procArrayGroupMemberXid = latestXid; - while (true) { - nextidx = pg_atomic_read_u32(&g_instance.proc_base->procArrayGroupFirst); - pg_atomic_write_u32(&proc->procArrayGroupNext, nextidx); - - if (pg_atomic_compare_exchange_u32( - &g_instance.proc_base->procArrayGroupFirst, &nextidx, (uint32)proc->pgprocno)) - break; + if (isSubTransaction) { + SetProcArrayGroupCache(proc, subTranactionXid, nSubTransactionXids, subTransactionXids, subTransactionLatestXid); + } else { + SetProcArrayGroupCache(proc, latestXid, 0, NULL, InvalidTransactionId); } + /* add current proc into ProcArrayGroup */ + ProcInsertIntoGroup(proc, &nextidx); + /* * If the list was not empty, the leader will clear our XID. It is * impossible to have followers without a leader because the first process @@ -685,20 +719,45 @@ static void ProcArrayGroupClearXid(PGPROC* proc, TransactionId latestXid) while (nextidx != INVALID_PGPROCNO) { PGPROC* proc_member = g_instance.proc_base_all_procs[nextidx]; PGXACT* pgxact = &g_instance.proc_base_all_xacts[nextidx]; + ereport(DEBUG2, (errmsg("handle group member from procArrayGroup, procno = %u, " + "procArrayGroupMemberXid = " XID_FMT ", " + "procArrayGroupSubXactNXids = %d, " + "procArrayGroupSubXactLatestXid = " XID_FMT ", " + "procArrayGroupNext = %u", + proc_member->pgprocno, + proc_member->procArrayGroupMemberXid, + proc_member->procArrayGroupSubXactNXids, + proc_member->procArrayGroupSubXactLatestXid, + proc_member->procArrayGroupNext))); + + /* + * If the proc_member is a transaction, perform ProcArrayEndTransactionInternal + * to clear its XID. If the proc_member is a subtransaction, + * perform XidCacheRemoveRunningXids to clear its XIDs and + * its committed subtransaction's XIDS. + * + * proc_member->procArrayGroupSubXactLatestXid !=0 when the group member + * is a subtransaction. + */ + if (proc_member->procArrayGroupSubXactLatestXid != InvalidTransactionId) { + XidCacheRemoveRunningXids(proc_member, pgxact); + } else { + groupMemberHasTransaction = true; + ProcArrayEndTransactionInternal( + proc_member, pgxact, proc_member->procArrayGroupMemberXid, &xid[index], &nsubxids[index]); + } - /* Don't need to update csn each loop, just update once after the loop. */ - commitcsn[index] = proc_member->commitCSN; - if (proc_member->commitCSN > maxcsn) - maxcsn = proc_member->commitCSN; - ProcArrayEndTransactionInternal( - proc_member, pgxact, proc_member->procArrayGroupMemberXid, &xid[index], &nsubxids[index]); /* Move to next proc in list. */ nextidx = pg_atomic_read_u32(&proc_member->procArrayGroupNext); index++; } - /* already hold lock, caculate snapshot after last invocation */ - CalculateLocalLatestSnapshot(false); + /* Already hold lock, caculate snapshot after last invocation, + * if there is at least one transaction in group. + */ + if (groupMemberHasTransaction) { + CalculateLocalLatestSnapshot(false); + } /* We're done with the lock now. */ LWLockRelease(ProcArrayLock); @@ -720,7 +779,7 @@ static void ProcArrayGroupClearXid(PGPROC* proc, TransactionId latestXid) /* ensure all previous writes are visible before follower continues. */ pg_write_barrier(); - proc_member->procArrayGroupMember = false; + ClearProcArrayGroupCache(proc_member); if (proc_member != t_thrd.proc) PGSemaphoreUnlock(&proc_member->sem); @@ -3848,35 +3907,31 @@ void ProcArrayGetReplicationSlotXmin(TransactionId* xmin, TransactionId* catalog LWLockRelease(ProcArrayLock); } -#define XidCacheRemove(i) \ - do { \ - t_thrd.proc->subxids.xids[i] = t_thrd.proc->subxids.xids[t_thrd.pgxact->nxids - 1]; \ - t_thrd.pgxact->nxids--; \ - } while (0) - /* * XidCacheRemoveRunningXids * * Remove a bunch of TransactionIds from the list of known-running * subtransactions for my backend. Both the specified xid and those in * the xids[] array (of length nxids) are removed from the subxids cache. - * latestXid must be the latest XID among the group. + * latestXid must be the latest XID among the group. We should store the + * required parameters into proc before performing XidCacheRemoveRunningXids, + * including subtransaction xid, the number of committed subtransaction, + * committed substransaction list, the latestXid between its xid and its + * committed subtransactions'. + * + * We don't do any locking here; caller must get the procArrayLock before + * perform XidCacheRemoveRunningXids. */ -void XidCacheRemoveRunningXids(TransactionId xid, int nxids, const TransactionId* xids, TransactionId latestXid) +void XidCacheRemoveRunningXids(PGPROC* proc, PGXACT* pgxact) { int i, j; + TransactionId xid = proc->procArrayGroupMemberXid; + int nxids = proc->procArrayGroupSubXactNXids; + TransactionId* xids = proc->procArrayGroupSubXactXids; + TransactionId latestXid = proc->procArrayGroupSubXactLatestXid; Assert(TransactionIdIsValid(xid)); - /* - * We must hold ProcArrayLock exclusively in order to remove transactions - * from the PGPROC array. (See src/backend/access/transam/README.) It's - * possible this could be relaxed since we know this routine is only used - * to abort subtransactions, but pending closer analysis we'd best be - * conservative. - */ - LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE); - /* * Under normal circumstances xid and xids[] will be in increasing order, * as will be the entries in subxids. Scan backwards to avoid O(N^2) @@ -3885,9 +3940,10 @@ void XidCacheRemoveRunningXids(TransactionId xid, int nxids, const TransactionId for (i = nxids - 1; i >= 0; i--) { TransactionId anxid = xids[i]; - for (j = t_thrd.pgxact->nxids - 1; j >= 0; j--) { - if (TransactionIdEquals(t_thrd.proc->subxids.xids[j], anxid)) { - XidCacheRemove(j); + for (j = pgxact->nxids - 1; j >= 0; j--) { + if (TransactionIdEquals(proc->subxids.xids[j], anxid)) { + proc->subxids.xids[j] = proc->subxids.xids[pgxact->nxids - 1]; + pgxact->nxids--; break; } } @@ -3903,9 +3959,10 @@ void XidCacheRemoveRunningXids(TransactionId xid, int nxids, const TransactionId ereport(WARNING, (errmsg("did not find subXID " XID_FMT " in t_thrd.proc", anxid))); } - for (j = t_thrd.pgxact->nxids - 1; j >= 0; j--) { - if (TransactionIdEquals(t_thrd.proc->subxids.xids[j], xid)) { - XidCacheRemove(j); + for (j = pgxact->nxids - 1; j >= 0; j--) { + if (TransactionIdEquals(proc->subxids.xids[j], xid)) { + proc->subxids.xids[j] = proc->subxids.xids[pgxact->nxids - 1]; + pgxact->nxids--; break; } } @@ -3918,7 +3975,6 @@ void XidCacheRemoveRunningXids(TransactionId xid, int nxids, const TransactionId if (TransactionIdPrecedes(t_thrd.xact_cxt.ShmemVariableCache->latestCompletedXid, latestXid)) t_thrd.xact_cxt.ShmemVariableCache->latestCompletedXid = latestXid; - LWLockRelease(ProcArrayLock); } #ifdef XIDCACHE_DEBUG diff --git a/src/gausskernel/storage/lmgr/proc.cpp b/src/gausskernel/storage/lmgr/proc.cpp index ffc6e4d48..f2c6cac0c 100755 --- a/src/gausskernel/storage/lmgr/proc.cpp +++ b/src/gausskernel/storage/lmgr/proc.cpp @@ -905,6 +905,9 @@ void InitProcess(void) /* Initialize fields for group XID clearing. */ t_thrd.proc->procArrayGroupMember = false; t_thrd.proc->procArrayGroupMemberXid = InvalidTransactionId; + t_thrd.proc->procArrayGroupSubXactNXids = InvalidTransactionId; + t_thrd.proc->procArrayGroupSubXactXids = NULL; + t_thrd.proc->procArrayGroupSubXactLatestXid = InvalidTransactionId; pg_atomic_init_u32(&t_thrd.proc->procArrayGroupNext, INVALID_PGPROCNO); /* Initialize fields for group snapshot getting. */ diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h index d52aa7775..bc122ff3f 100755 --- a/src/include/storage/proc.h +++ b/src/include/storage/proc.h @@ -214,6 +214,18 @@ struct PGPROC { */ TransactionId procArrayGroupMemberXid; + int procArrayGroupSubXactNXids; + + TransactionId* procArrayGroupSubXactXids; + + /* + * lastestXid amoung subtransaction's xid and it's committed children's, + * which can be detemined whether the group member is a subtransaction or + * transaction. procArrayGroupSubXactLatestXid != 0 only when the group + * memeber is subtransaction. + */ + TransactionId procArrayGroupSubXactLatestXid; + /* Support for group snapshot getting. */ bool snapshotGroupMember; /* next ProcArray group member waiting for snapshot getting */ diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h index 5a8370506..91f114628 100755 --- a/src/include/storage/procarray.h +++ b/src/include/storage/procarray.h @@ -125,9 +125,13 @@ extern void CancelSingleNodeBackends(Oid databaseOid, Oid userOid, ProcSignalRea extern int CountUserBackends(Oid roleid); extern bool CountOtherDBBackends(Oid databaseId, int* nbackends, int* nprepared); -extern void XidCacheRemoveRunningXids(TransactionId xid, int nxids, const TransactionId* xids, TransactionId latestXid); +extern void XidCacheRemoveRunningXids(PGPROC* proc, PGXACT* pgxact); extern void SetPgXactXidInvalid(void); +extern void ProcArrayGroupClearXid(bool isSubTransaction, PGPROC* proc, TransactionId latestXid, + TransactionId subTranactionXid, int nSubTransactionXids, + TransactionId* subTransactionXids, TransactionId subTransactionLatestXid); + extern void ProcArraySetReplicationSlotXmin(TransactionId xmin, TransactionId catalog_xmin, bool already_locked); extern void ProcArrayGetReplicationSlotXmin(TransactionId* xmin, TransactionId* catalog_xmin); From 330bc6263104284f6bac8897924d44825a23e4ed Mon Sep 17 00:00:00 2001 From: luo_zihao5524 Date: Tue, 11 Jul 2023 20:47:54 +0800 Subject: [PATCH 037/304] =?UTF-8?q?=E4=BD=BF=E7=94=A8=E6=96=B0=E5=8F=82?= =?UTF-8?q?=E6=95=B0=E6=8E=A7=E5=88=B6A=E5=85=BC=E5=AE=B9=E6=80=A7?= =?UTF-8?q?=E4=B8=8B=E7=A9=BA=E5=AD=97=E7=AC=A6=E4=B8=B2=E6=84=8F=E4=B9=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/parser/gram.y | 2 +- src/common/backend/parser/hint_gram.y | 2 +- src/common/backend/utils/adt/a_compat.cpp | 29 +- src/common/backend/utils/adt/format_type.cpp | 3 +- src/common/backend/utils/adt/nlssort.cpp | 3 +- src/common/backend/utils/adt/regexp.cpp | 12 +- src/common/backend/utils/adt/varchar.cpp | 2 +- src/common/backend/utils/adt/varlena.cpp | 81 +- src/common/backend/utils/misc/guc/guc_sql.cpp | 3 +- src/gausskernel/optimizer/commands/copy.cpp | 4 +- src/gausskernel/process/tcop/postgres.cpp | 13 +- .../codegen/codegenutil/varlenacodegen.cpp | 18 +- .../codegen/vecexecutor/vecexprcodegen.cpp | 6 +- src/gausskernel/runtime/opfusion/opfusion.cpp | 4 +- .../vecexecutor/vecprimitive/varchar.inl | 2 +- src/include/miscadmin.h | 4 +- .../regress/expected/accept_empty_str.out | 1815 ++++++++++++++++ .../regress/expected/not_accept_empty_str.out | 1822 +++++++++++++++++ src/test/regress/expected/pg_empty_str.out | 1817 ++++++++++++++++ .../regress/input/accept_empty_copy.source | 33 + .../input/not_accept_empty_copy.source | 33 + .../regress/output/accept_empty_copy.source | 107 + .../output/not_accept_empty_copy.source | 107 + src/test/regress/parallel_schedule0A | 3 + src/test/regress/sql/accept_empty_str.sql | 405 ++++ src/test/regress/sql/not_accept_empty_str.sql | 405 ++++ src/test/regress/sql/pg_empty_str.sql | 407 ++++ 27 files changed, 7069 insertions(+), 73 deletions(-) create mode 100644 src/test/regress/expected/accept_empty_str.out create mode 100644 src/test/regress/expected/not_accept_empty_str.out create mode 100644 src/test/regress/expected/pg_empty_str.out create mode 100644 src/test/regress/input/accept_empty_copy.source create mode 100644 src/test/regress/input/not_accept_empty_copy.source create mode 100644 src/test/regress/output/accept_empty_copy.source create mode 100644 src/test/regress/output/not_accept_empty_copy.source create mode 100644 src/test/regress/sql/accept_empty_str.sql create mode 100644 src/test/regress/sql/not_accept_empty_str.sql create mode 100644 src/test/regress/sql/pg_empty_str.sql diff --git a/src/common/backend/parser/gram.y b/src/common/backend/parser/gram.y index a105fb82a..0fa2efeec 100644 --- a/src/common/backend/parser/gram.y +++ b/src/common/backend/parser/gram.y @@ -29717,7 +29717,7 @@ makeStringConst(char *str, int location) if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) { - if (NULL == str || 0 == strlen(str)) + if (NULL == str || (0 == strlen(str) && !ACCEPT_EMPTY_STR)) { n->val.type = T_Null; n->val.val.str = str; diff --git a/src/common/backend/parser/hint_gram.y b/src/common/backend/parser/hint_gram.y index 19af74265..37079d9f2 100755 --- a/src/common/backend/parser/hint_gram.y +++ b/src/common/backend/parser/hint_gram.y @@ -681,7 +681,7 @@ makeStringValue(char *str) if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) { - if (NULL == str || 0 == strlen(str)) + if (NULL == str || (0 == strlen(str) && !ACCEPT_EMPTY_STR)) { val->type = T_Null; val->val.str = str; diff --git a/src/common/backend/utils/adt/a_compat.cpp b/src/common/backend/utils/adt/a_compat.cpp index 00ef60dcf..eb99d69f3 100644 --- a/src/common/backend/utils/adt/a_compat.cpp +++ b/src/common/backend/utils/adt/a_compat.cpp @@ -204,7 +204,8 @@ Datum lpad(PG_FUNCTION_ARGS) SET_VARSIZE(ret, ptr_ret - (char*)ret); - if (0 == VARSIZE_ANY_EXHDR(ret) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !RETURN_NS) + if (0 == VARSIZE_ANY_EXHDR(ret) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && + !ACCEPT_EMPTY_STR && !RETURN_NS) PG_RETURN_NULL(); else PG_RETURN_TEXT_P(ret); @@ -300,7 +301,8 @@ Datum rpad(PG_FUNCTION_ARGS) SET_VARSIZE(ret, ptr_ret - (char*)ret); - if (0 == VARSIZE_ANY_EXHDR(ret) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !RETURN_NS) + if (0 == VARSIZE_ANY_EXHDR(ret) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && + !ACCEPT_EMPTY_STR && !RETURN_NS) PG_RETURN_NULL(); else PG_RETURN_TEXT_P(ret); @@ -331,7 +333,8 @@ Datum btrim(PG_FUNCTION_ARGS) ret = dotrim(VARDATA_ANY(string), VARSIZE_ANY_EXHDR(string), VARDATA_ANY(set), VARSIZE_ANY_EXHDR(set), true, true); - if ((ret == NULL || 0 == VARSIZE_ANY_EXHDR(ret)) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) + if ((ret == NULL || 0 == VARSIZE_ANY_EXHDR(ret)) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && + !ACCEPT_EMPTY_STR) PG_RETURN_NULL(); else PG_RETURN_TEXT_P(ret); @@ -352,7 +355,8 @@ Datum btrim1(PG_FUNCTION_ARGS) ret = dotrim(VARDATA_ANY(string), VARSIZE_ANY_EXHDR(string), " ", 1, true, true); - if ((ret == NULL || 0 == VARSIZE_ANY_EXHDR(ret)) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) + if ((ret == NULL || 0 == VARSIZE_ANY_EXHDR(ret)) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && + !ACCEPT_EMPTY_STR) PG_RETURN_NULL(); else PG_RETURN_TEXT_P(ret); @@ -589,7 +593,8 @@ Datum ltrim(PG_FUNCTION_ARGS) ret = dotrim(VARDATA_ANY(string), VARSIZE_ANY_EXHDR(string), VARDATA_ANY(set), VARSIZE_ANY_EXHDR(set), true, false); - if ((ret == NULL || 0 == VARSIZE_ANY_EXHDR(ret)) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) + if ((ret == NULL || 0 == VARSIZE_ANY_EXHDR(ret)) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && + !ACCEPT_EMPTY_STR) PG_RETURN_NULL(); else PG_RETURN_TEXT_P(ret); @@ -609,7 +614,8 @@ Datum ltrim1(PG_FUNCTION_ARGS) ret = dotrim(VARDATA_ANY(string), VARSIZE_ANY_EXHDR(string), " ", 1, true, false); - if ((ret == NULL || 0 == VARSIZE_ANY_EXHDR(ret)) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) + if ((ret == NULL || 0 == VARSIZE_ANY_EXHDR(ret)) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && + !ACCEPT_EMPTY_STR) PG_RETURN_NULL(); else PG_RETURN_TEXT_P(ret); @@ -639,7 +645,8 @@ Datum rtrim(PG_FUNCTION_ARGS) ret = dotrim(VARDATA_ANY(string), VARSIZE_ANY_EXHDR(string), VARDATA_ANY(set), VARSIZE_ANY_EXHDR(set), false, true); - if ((ret == NULL || 0 == VARSIZE_ANY_EXHDR(ret)) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) + if ((ret == NULL || 0 == VARSIZE_ANY_EXHDR(ret)) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && + !ACCEPT_EMPTY_STR) PG_RETURN_NULL(); else PG_RETURN_TEXT_P(ret); @@ -668,7 +675,8 @@ Datum rtrim1(PG_FUNCTION_ARGS) ret = dotrim(VARDATA_ANY(string), VARSIZE_ANY_EXHDR(string), " ", 1, false, true); } - if ((ret == NULL || 0 == VARSIZE_ANY_EXHDR(ret)) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) + if ((ret == NULL || 0 == VARSIZE_ANY_EXHDR(ret)) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && + !ACCEPT_EMPTY_STR) PG_RETURN_NULL(); else PG_RETURN_TEXT_P(ret); @@ -775,7 +783,7 @@ Datum translate(PG_FUNCTION_ARGS) m -= source_len; } - if (0 == retlen && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) { + if (0 == retlen && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !ACCEPT_EMPTY_STR) { PG_RETURN_NULL(); } else { SET_VARSIZE(result, retlen + VARHDRSZ); @@ -999,7 +1007,8 @@ Datum repeat(PG_FUNCTION_ARGS) tlen -= slen; } - if (0 == VARSIZE_ANY_EXHDR(result) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !RETURN_NS) { + if (0 == VARSIZE_ANY_EXHDR(result) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && + !ACCEPT_EMPTY_STR && !RETURN_NS) { PG_RETURN_NULL(); } else PG_RETURN_TEXT_P(result); diff --git a/src/common/backend/utils/adt/format_type.cpp b/src/common/backend/utils/adt/format_type.cpp index c4e22cf9b..e6a40bd67 100644 --- a/src/common/backend/utils/adt/format_type.cpp +++ b/src/common/backend/utils/adt/format_type.cpp @@ -461,7 +461,8 @@ Datum oidvectortypes(PG_FUNCTION_ARGS) left -= slen; } - if ((!strcmp(result, "")) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !RETURN_NS) { + if ((!strcmp(result, "")) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && + !ACCEPT_EMPTY_STR && !RETURN_NS) { pfree_ext(result); PG_RETURN_NULL(); } diff --git a/src/common/backend/utils/adt/nlssort.cpp b/src/common/backend/utils/adt/nlssort.cpp index a0ddfb22f..aae2a9b76 100644 --- a/src/common/backend/utils/adt/nlssort.cpp +++ b/src/common/backend/utils/adt/nlssort.cpp @@ -23,6 +23,7 @@ * --------------------------------------------------------------------------------------- */ #include "postgres.h" +#include "miscadmin.h" #include "mb/pg_wchar.h" #include "utils/builtins.h" #include "knl/knl_session.h" @@ -71,7 +72,7 @@ Datum nlssort(PG_FUNCTION_ARGS) PG_RETURN_NULL(); } if (VARSIZE_ANY_EXHDR(PG_GETARG_TEXT_P(0)) == 0) { - if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) { + if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !ACCEPT_EMPTY_STR) { PG_RETURN_NULL(); } else { PG_RETURN_TEXT_P(cstring_to_text("\0")); diff --git a/src/common/backend/utils/adt/regexp.cpp b/src/common/backend/utils/adt/regexp.cpp index 692b59a0a..c9798130d 100644 --- a/src/common/backend/utils/adt/regexp.cpp +++ b/src/common/backend/utils/adt/regexp.cpp @@ -613,7 +613,7 @@ Datum regexp_replace(PG_FUNCTION_ARGS) re = RE_compile_and_cache(pattern, re_flags.cflags, PG_GET_COLLATION()); result = replace_text_regexp(src, (void*)re, r, position, occurrence); - if (VARHDRSZ == VARSIZE(result) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) + if (VARHDRSZ == VARSIZE(result) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !ACCEPT_EMPTY_STR) PG_RETURN_NULL(); else PG_RETURN_TEXT_P(result); @@ -684,7 +684,7 @@ Datum textregexreplace_noopt(PG_FUNCTION_ARGS) result = replace_text_regexp(s, (void*)re, r, 1, occurrence); - if (VARHDRSZ == VARSIZE(result) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) + if (VARHDRSZ == VARSIZE(result) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !ACCEPT_EMPTY_STR) PG_RETURN_NULL(); else PG_RETURN_TEXT_P(result); @@ -739,7 +739,7 @@ Datum textregexreplace(PG_FUNCTION_ARGS) result = replace_text_regexp(s, (void*)re, r, 1, occurrence); - if (VARHDRSZ == VARSIZE(result) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) + if (VARHDRSZ == VARSIZE(result) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !ACCEPT_EMPTY_STR) PG_RETURN_NULL(); else PG_RETURN_TEXT_P(result); @@ -1326,13 +1326,13 @@ static ArrayType* build_regexp_matches_result(regexp_matches_ctx* matchctx) /* return value datatype must be text */ #define RESET_NULL_FLAG(_result) \ do { \ - if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !RETURN_NS) { \ + if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !RETURN_NS && !ACCEPT_EMPTY_STR) { \ if ((_result) == ((Datum)0)) { \ fcinfo->isnull = true; \ } else { \ text *t = DatumGetTextP((_result)); \ fcinfo->isnull = false; \ - if (VARSIZE_ANY_EXHDR(t) == 0) { \ + if (VARSIZE_ANY_EXHDR(t) == 0 && !ACCEPT_EMPTY_STR) { \ fcinfo->isnull = true; \ (_result) = (Datum)0; \ } \ @@ -1739,7 +1739,7 @@ Datum regexp_substr_core(PG_FUNCTION_ARGS) PG_GET_COLLATION()); if (ret == NULL || (VARHDRSZ == VARSIZE(ret) && - u_sess->attr.attr_sql.sql_compatibility == A_FORMAT)) { + u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !ACCEPT_EMPTY_STR)) { PG_RETURN_NULL(); } else { PG_RETURN_TEXT_P(ret); diff --git a/src/common/backend/utils/adt/varchar.cpp b/src/common/backend/utils/adt/varchar.cpp index 4188e7a60..00def681a 100644 --- a/src/common/backend/utils/adt/varchar.cpp +++ b/src/common/backend/utils/adt/varchar.cpp @@ -1595,7 +1595,7 @@ static void vlpad_internal(ScalarVector* parg1, ScalarVector* parg2, ScalarVecto SET_VARSIZE(ret, ptr_ret - (char*)ret); - if (0 == VARSIZE_ANY_EXHDR(ret) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !RETURN_NS) { + if (0 == VARSIZE_ANY_EXHDR(ret) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !ACCEPT_EMPTY_STR && !RETURN_NS) { SET_NULL(pflagsRes[idx]); } else { VecRet->m_vals[idx] = PointerGetDatum(ret); diff --git a/src/common/backend/utils/adt/varlena.cpp b/src/common/backend/utils/adt/varlena.cpp index 0b929a3ab..0fa127dbf 100644 --- a/src/common/backend/utils/adt/varlena.cpp +++ b/src/common/backend/utils/adt/varlena.cpp @@ -141,6 +141,8 @@ static void text_format_append_string(StringInfo buf, const char* str, int flags // adapt A db's substrb static text* get_substring_really(Datum str, int32 start, int32 length, bool length_not_specified); +static text* get_result_of_concat(text* result, FunctionCallInfo fcinfo); + /***************************************************************************** * CONVERSION ROUTINES EXPORTED FOR USE BY C CODE * *****************************************************************************/ @@ -3311,7 +3313,7 @@ Datum bytea_substr_orclcompat(PG_FUNCTION_ARGS) total = toast_raw_datum_size(str) - VARHDRSZ; if ((length < 0) || (start > total) || (start + total < 0)) { - if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT || + if ((u_sess->attr.attr_sql.sql_compatibility == A_FORMAT&& !ACCEPT_EMPTY_STR) || u_sess->attr.attr_sql.sql_compatibility == B_FORMAT) PG_RETURN_NULL(); else { @@ -3325,10 +3327,12 @@ Datum bytea_substr_orclcompat(PG_FUNCTION_ARGS) */ result = bytea_substring_orclcompat(str, start, length, false); - if ((NULL == result || 0 == VARSIZE_ANY_EXHDR(result)) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) + if (NULL == result || (0 == VARSIZE_ANY_EXHDR(result) && + u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !ACCEPT_EMPTY_STR)) { PG_RETURN_NULL(); - else - PG_RETURN_BYTEA_P(result); + } + + PG_RETURN_BYTEA_P(result); } // adapt A db's substr(bytea x,integer y) @@ -3343,7 +3347,7 @@ Datum bytea_substr_no_len_orclcompat(PG_FUNCTION_ARGS) total = toast_raw_datum_size(str) - VARHDRSZ; if ((start > total) || (start + total < 0)) { - if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) + if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !ACCEPT_EMPTY_STR) PG_RETURN_NULL(); else { result = PG_STR_GET_BYTEA(""); @@ -3356,10 +3360,12 @@ Datum bytea_substr_no_len_orclcompat(PG_FUNCTION_ARGS) */ result = bytea_substring_orclcompat(str, start, -1, true); - if ((NULL == result || 0 == VARSIZE_ANY_EXHDR(result)) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) + if (NULL == result || (0 == VARSIZE_ANY_EXHDR(result) && + u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !ACCEPT_EMPTY_STR)) { PG_RETURN_NULL(); - else - PG_RETURN_BYTEA_P(result); + } + + PG_RETURN_BYTEA_P(result); } // Does the real work for bytea_substr_orclcompat() and bytea_substr_no_len_orclcompat(). @@ -4424,10 +4430,12 @@ Datum replace_text(PG_FUNCTION_ARGS) ret_text = cstring_to_text_with_len(str.data, str.len); pfree_ext(str.data); - if (VARHDRSZ == VARSIZE(ret_text) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) + if (VARHDRSZ == VARSIZE(ret_text) && + u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !ACCEPT_EMPTY_STR) { PG_RETURN_NULL(); - else - PG_RETURN_TEXT_P(ret_text); + } + + PG_RETURN_TEXT_P(ret_text); } /* @@ -4715,7 +4723,7 @@ Datum split_text(PG_FUNCTION_ARGS) if (inputstring_len < 1) { text_position_cleanup(&state); - if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !RETURN_NS) { + if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !ACCEPT_EMPTY_STR && !RETURN_NS) { PG_RETURN_NULL(); } @@ -4730,7 +4738,7 @@ Datum split_text(PG_FUNCTION_ARGS) PG_RETURN_TEXT_P(inputstring); } - if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !RETURN_NS) { + if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !ACCEPT_EMPTY_STR && !RETURN_NS) { PG_RETURN_NULL(); } @@ -4749,7 +4757,7 @@ Datum split_text(PG_FUNCTION_ARGS) PG_RETURN_TEXT_P(inputstring); } - if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) { + if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !ACCEPT_EMPTY_STR) { PG_RETURN_NULL(); } @@ -4776,7 +4784,7 @@ Datum split_text(PG_FUNCTION_ARGS) result_text = text_substring(PointerGetDatum(inputstring), start_posn, end_posn - start_posn, false); } - if (TEXTISORANULL(result_text) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) { + if (TEXTISORANULL(result_text) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !ACCEPT_EMPTY_STR) { PG_RETURN_NULL(); } @@ -4968,7 +4976,7 @@ Datum array_to_text(PG_FUNCTION_ARGS) result = array_to_text_internal(fcinfo, v, fldsep, NULL); /* To A db, empty string need return NULL.*/ - if (0 == VARSIZE_ANY_EXHDR(result) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) { + if (0 == VARSIZE_ANY_EXHDR(result) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !ACCEPT_EMPTY_STR) { PG_RETURN_NULL(); } else { PG_RETURN_TEXT_P(result); @@ -5005,7 +5013,7 @@ Datum array_to_text_null(PG_FUNCTION_ARGS) result = array_to_text_internal(fcinfo, v, fldsep, null_string); /* To A db, empty string need return NULL.*/ - if (0 == VARSIZE_ANY_EXHDR(result) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) { + if (0 == VARSIZE_ANY_EXHDR(result) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !ACCEPT_EMPTY_STR) { PG_RETURN_NULL(); } else { PG_RETURN_TEXT_P(result); @@ -6335,11 +6343,25 @@ static text* concat_internal(const char* sepstr, int seplen, int argidx, Functio result = cstring_to_text_with_len(str.data, str.len); pfree_ext(str.data); - if ((result == NULL || (0 == VARSIZE_ANY_EXHDR(result) && !DB_IS_CMPT(B_FORMAT | PG_FORMAT))) && - (CONCAT_VARIADIC || DB_IS_CMPT(A_FORMAT))) + return get_result_of_concat(result, fcinfo); +} + +static text* get_result_of_concat(text* result, FunctionCallInfo fcinfo) +{ + if (result == NULL) { PG_RETURN_NULL(); - else + } + + if (VARSIZE_ANY_EXHDR(result) > 0 || + DB_IS_CMPT(B_FORMAT | PG_FORMAT) || + (DB_IS_CMPT(A_FORMAT) && ACCEPT_EMPTY_STR)) { return result; + } + + if (DB_IS_CMPT(A_FORMAT) || CONCAT_VARIADIC) { + PG_RETURN_NULL(); + } + return result; } /* @@ -6458,7 +6480,7 @@ Datum text_left(PG_FUNCTION_ARGS) } rlen = pg_mbcharcliplen(p, len, part_off); - if (0 == rlen && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) { + if (0 == rlen && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !ACCEPT_EMPTY_STR) { PG_RETURN_NULL(); } @@ -6495,7 +6517,7 @@ Datum text_right(PG_FUNCTION_ARGS) } } off = pg_mbcharcliplen(p, len, part_off); - if (0 == (len - off) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) { + if (0 == (len - off) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !ACCEPT_EMPTY_STR) { PG_RETURN_NULL(); } @@ -6794,7 +6816,8 @@ Datum text_format(PG_FUNCTION_ARGS) result = cstring_to_text_with_len(str.data, str.len); pfree_ext(str.data); - if ((result == NULL || VARSIZE_ANY_EXHDR(result) == 0) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) + if (result == NULL || + (VARSIZE_ANY_EXHDR(result) == 0 && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !ACCEPT_EMPTY_STR)) PG_RETURN_NULL(); else PG_RETURN_TEXT_P(result); @@ -7160,7 +7183,7 @@ Datum substrb_with_lenth(PG_FUNCTION_ARGS) int32 total = 0; total = toast_raw_datum_size(str) - VARHDRSZ; if ((length < 0) || (total == 0) || (start > total) || (start + total < 0)) { - if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) + if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !ACCEPT_EMPTY_STR) PG_RETURN_NULL(); else { result = cstring_to_text(""); @@ -7169,8 +7192,10 @@ Datum substrb_with_lenth(PG_FUNCTION_ARGS) } result = get_substring_really(str, start, length, false); - if ((NULL == result || 0 == VARSIZE_ANY_EXHDR(result)) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) + if ((NULL == result || 0 == VARSIZE_ANY_EXHDR(result)) && + u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !ACCEPT_EMPTY_STR) { PG_RETURN_NULL(); + } PG_RETURN_TEXT_P(result); } @@ -7185,7 +7210,7 @@ Datum substrb_without_lenth(PG_FUNCTION_ARGS) int32 total = 0; total = toast_raw_datum_size(str) - VARHDRSZ; if ((total == 0) || (start > total) || (start + total < 0)) { - if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) + if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !ACCEPT_EMPTY_STR) PG_RETURN_NULL(); else { result = cstring_to_text(""); @@ -7194,8 +7219,10 @@ Datum substrb_without_lenth(PG_FUNCTION_ARGS) } result = get_substring_really(str, start, -1, true); - if ((NULL == result || 0 == VARSIZE_ANY_EXHDR(result)) && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) + if ((NULL == result || 0 == VARSIZE_ANY_EXHDR(result)) && + u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !ACCEPT_EMPTY_STR) { PG_RETURN_NULL(); + } PG_RETURN_TEXT_P(result); } diff --git a/src/common/backend/utils/misc/guc/guc_sql.cpp b/src/common/backend/utils/misc/guc/guc_sql.cpp index eea75db5a..ce1a49ba6 100755 --- a/src/common/backend/utils/misc/guc/guc_sql.cpp +++ b/src/common/backend/utils/misc/guc/guc_sql.cpp @@ -383,7 +383,8 @@ static const struct behavior_compat_entry behavior_compat_options[OPT_MAX] = { {"pgformat_substr", OPT_PGFORMAT_SUBSTR}, {"truncate_numeric_tail_zero", OPT_TRUNC_NUMERIC_TAIL_ZERO}, {"allow_orderby_undistinct_column", OPT_ALLOW_ORDERBY_UNDISTINCT_COLUMN}, - {"select_into_return_null", OPT_SELECT_INTO_RETURN_NULL} + {"select_into_return_null", OPT_SELECT_INTO_RETURN_NULL}, + {"accept_empty_str", OPT_ACCEPT_EMPTY_STR} }; // increase SQL_IGNORE_STRATEGY_NUM if we need more strategy diff --git a/src/gausskernel/optimizer/commands/copy.cpp b/src/gausskernel/optimizer/commands/copy.cpp index 74ba6209e..8d404b1e2 100644 --- a/src/gausskernel/optimizer/commands/copy.cpp +++ b/src/gausskernel/optimizer/commands/copy.cpp @@ -6445,8 +6445,8 @@ bool NextCopyFrom(CopyState cstate, ExprContext* econtext, Datum* values, bool* * 1. A db SQL compatibility requires; or * 2. This column donesn't accept any empty string. */ - if ((u_sess->attr.attr_sql.sql_compatibility == A_FORMAT || !accept_empty_str[m]) && - (string != NULL && string[0] == '\0')) { + if (((u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !ACCEPT_EMPTY_STR) || + !accept_empty_str[m]) && (string != NULL && string[0] == '\0')) { /* for any type, '' = null */ string = NULL; } diff --git a/src/gausskernel/process/tcop/postgres.cpp b/src/gausskernel/process/tcop/postgres.cpp index 7ca0b86e6..93dc3c597 100755 --- a/src/gausskernel/process/tcop/postgres.cpp +++ b/src/gausskernel/process/tcop/postgres.cpp @@ -4040,8 +4040,8 @@ static int getSingleNodeIdx(StringInfo input_message, CachedPlanSource* psrc, co plength = pq_getmsgint(input_message, 4); isNull = (plength == -1); /* add null value process for date type */ - if ((VARCHAROID == ptype || TIMESTAMPOID == ptype || TIMESTAMPTZOID == ptype || TIMEOID == ptype || - TIMETZOID == ptype || INTERVALOID == ptype || SMALLDATETIMEOID == ptype) && + if (((VARCHAROID == ptype && !ACCEPT_EMPTY_STR) || TIMESTAMPOID == ptype || TIMESTAMPTZOID == ptype || + TIMEOID == ptype || TIMETZOID == ptype || INTERVALOID == ptype || SMALLDATETIMEOID == ptype) && 0 == plength && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) isNull = true; @@ -4838,8 +4838,8 @@ static void exec_bind_message(StringInfo input_message) plength = pq_getmsgint(input_message, 4); isNull = (plength == -1); /* add null value process for date type */ - if ((VARCHAROID == ptype || TIMESTAMPOID == ptype || TIMESTAMPTZOID == ptype || TIMEOID == ptype || - TIMETZOID == ptype || INTERVALOID == ptype || SMALLDATETIMEOID == ptype) && + if (((VARCHAROID == ptype && !ACCEPT_EMPTY_STR) || TIMESTAMPOID == ptype || TIMESTAMPTZOID == ptype || + TIMEOID == ptype || TIMETZOID == ptype || INTERVALOID == ptype || SMALLDATETIMEOID == ptype) && 0 == plength && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) isNull = true; @@ -11542,8 +11542,9 @@ static void exec_batch_bind_execute(StringInfo input_message) plength = pq_getmsgint(input_message, 4); isNull = (plength == -1); /* add null value process for date type */ - if ((VARCHAROID == ptype || TIMESTAMPOID == ptype || TIMESTAMPTZOID == ptype || TIMEOID == ptype || - TIMETZOID == ptype || INTERVALOID == ptype || SMALLDATETIMEOID == ptype) && + if (((VARCHAROID == ptype && !ACCEPT_EMPTY_STR) || TIMESTAMPOID == ptype || + TIMESTAMPTZOID == ptype || TIMEOID == ptype || TIMETZOID == ptype || + INTERVALOID == ptype || SMALLDATETIMEOID == ptype) && 0 == plength && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) isNull = true; diff --git a/src/gausskernel/runtime/codegen/codegenutil/varlenacodegen.cpp b/src/gausskernel/runtime/codegen/codegenutil/varlenacodegen.cpp index 9b379250c..1c6eced61 100644 --- a/src/gausskernel/runtime/codegen/codegenutil/varlenacodegen.cpp +++ b/src/gausskernel/runtime/codegen/codegenutil/varlenacodegen.cpp @@ -450,7 +450,7 @@ llvm::Function* substr_codegen() * in case of A db compatible format we have to prepare a flag to indicate whether * result string is NULL, so we add one more parameter. */ - if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) + if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !ACCEPT_EMPTY_STR) fn_prototype.addArgument(GsCodeGen::NamedVariable("isNull", int8PtrType)); llvm::Function* jitted_substr = fn_prototype.generatePrototype(&builder, &llvmargs[0]); @@ -459,7 +459,7 @@ llvm::Function* substr_codegen() start = llvmargs[1]; len = llvmargs[2]; - if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) + if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !ACCEPT_EMPTY_STR) isNull = llvmargs[3]; /* @@ -497,7 +497,7 @@ llvm::Function* substr_codegen() *in case of ORC, we should set isNull to True if res_len == 0; *otherwise, just return the result. */ - if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) { + if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !ACCEPT_EMPTY_STR) { DEFINE_BLOCK(be_null, jitted_substr); DEFINE_BLOCK(bnot_null, jitted_substr); DEFINE_BLOCK(ret_bb, jitted_substr); @@ -568,13 +568,13 @@ llvm::Function* rtrim1_codegen() * in case of A db compatible format we have to prepare a flag to indicate whether * result string is NULL, so we add one more parameter. */ - if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) + if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !ACCEPT_EMPTY_STR) fn_prototype.addArgument(GsCodeGen::NamedVariable("isNull", int8PtrType)); llvm::Function* jitted_rtrim1 = fn_prototype.generatePrototype(&builder, &llvmargs[0]); argval = llvmargs[0]; - if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) + if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !ACCEPT_EMPTY_STR) isNull = llvmargs[1]; /* load rtrim1 ir function from module */ @@ -599,7 +599,7 @@ llvm::Function* rtrim1_codegen() *in case of ORC, we should set isNull to True if res_len == 0; *otherwise, just return the result. */ - if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) { + if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !ACCEPT_EMPTY_STR) { DEFINE_BLOCK(be_null, jitted_rtrim1); DEFINE_BLOCK(bnot_null, jitted_rtrim1); DEFINE_BLOCK(ret_bb, jitted_rtrim1); @@ -671,12 +671,12 @@ llvm::Function* btrim1_codegen() * in case of A db compatible format we have to prepare a flag to indicate whether * result string is NULL, so we add one more parameter. */ - if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) + if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !ACCEPT_EMPTY_STR) fn_prototype.addArgument(GsCodeGen::NamedVariable("isNull", int8PtrType)); llvm::Function* jitted_btrim1 = fn_prototype.generatePrototype(&builder, &llvmargs[0]); argval = llvmargs[0]; - if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) + if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !ACCEPT_EMPTY_STR) isNull = llvmargs[1]; /* load rtrim1 ir function from IR file*/ @@ -701,7 +701,7 @@ llvm::Function* btrim1_codegen() *in case of ORC, we should set isNull to True if res_len == 0; *otherwise, just return the result. */ - if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) { + if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !ACCEPT_EMPTY_STR) { DEFINE_BLOCK(be_null, jitted_btrim1); DEFINE_BLOCK(bnot_null, jitted_btrim1); DEFINE_BLOCK(ret_bb, jitted_btrim1); diff --git a/src/gausskernel/runtime/codegen/vecexecutor/vecexprcodegen.cpp b/src/gausskernel/runtime/codegen/vecexecutor/vecexprcodegen.cpp index 05487de92..a0f0e5f06 100644 --- a/src/gausskernel/runtime/codegen/vecexecutor/vecexprcodegen.cpp +++ b/src/gausskernel/runtime/codegen/vecexecutor/vecexprcodegen.cpp @@ -2874,7 +2874,7 @@ llvm::Value* VecExprCodeGen::FuncCodeGen(ExprCodeGenArgs* args) * then we have to set 'isNull' flag to True, * and then jump to the be_null block. */ - if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) { + if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !ACCEPT_EMPTY_STR) { res1 = inner_builder.CreateCall(func_substr, {inargs[0], inargs[1], inargs[2], isNull}); llvm::Value* res_flag = inner_builder.CreateLoad(int8Type, isNull, "resflag"); llvm::Value* cmp = inner_builder.CreateICmpEQ(res_flag, null_true, "check"); @@ -2897,7 +2897,7 @@ llvm::Value* VecExprCodeGen::FuncCodeGen(ExprCodeGenArgs* args) * then we have to set 'isNull' flag to True, * and then jump to the be_null block. */ - if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) { + if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !ACCEPT_EMPTY_STR) { res1 = inner_builder.CreateCall(func_rtrim1, {inargs[0], isNull}); llvm::Value* res_flag = inner_builder.CreateLoad(int8Type, isNull, "resflag"); llvm::Value* cmp = inner_builder.CreateICmpEQ(res_flag, null_true, "check"); @@ -2919,7 +2919,7 @@ llvm::Value* VecExprCodeGen::FuncCodeGen(ExprCodeGenArgs* args) * then we have to set 'isNull' flag to True, * and then jump to the be_null block. */ - if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) { + if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !ACCEPT_EMPTY_STR) { res1 = inner_builder.CreateCall(func_btrim1, {inargs[0], isNull}); llvm::Value* res_flag = inner_builder.CreateLoad(int8Type, isNull, "resflag"); llvm::Value* cmp = inner_builder.CreateICmpEQ(res_flag, null_true, "check"); diff --git a/src/gausskernel/runtime/opfusion/opfusion.cpp b/src/gausskernel/runtime/opfusion/opfusion.cpp index 55d88a634..5c1826385 100644 --- a/src/gausskernel/runtime/opfusion/opfusion.cpp +++ b/src/gausskernel/runtime/opfusion/opfusion.cpp @@ -816,8 +816,8 @@ void OpFusion::updatePreAllocParamter(StringInfo input_message) plength = pq_getmsgint(input_message, 4); isNull = (plength == -1); /* add null value process for date type */ - if ((VARCHAROID == ptype || TIMESTAMPOID == ptype || TIMESTAMPTZOID == ptype || TIMEOID == ptype || - TIMETZOID == ptype || INTERVALOID == ptype || SMALLDATETIMEOID == ptype) && + if (((VARCHAROID == ptype && !ACCEPT_EMPTY_STR) || TIMESTAMPOID == ptype || TIMESTAMPTZOID == ptype || + TIMEOID == ptype || TIMETZOID == ptype || INTERVALOID == ptype || SMALLDATETIMEOID == ptype) && plength == 0 && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) { isNull = true; } diff --git a/src/gausskernel/runtime/vecexecutor/vecprimitive/varchar.inl b/src/gausskernel/runtime/vecexecutor/vecprimitive/varchar.inl index e32e65e22..9d60cca42 100755 --- a/src/gausskernel/runtime/vecexecutor/vecprimitive/varchar.inl +++ b/src/gausskernel/runtime/vecexecutor/vecprimitive/varchar.inl @@ -309,7 +309,7 @@ inline static Datum null_return(bool *is_null) { text *result = NULL; - if(u_sess->attr.attr_sql.sql_compatibility == A_FORMAT) + if(u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && !ACCEPT_EMPTY_STR) { *is_null = true; return (Datum)0; diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h index 5dce1abc4..c5ac8b9dd 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h @@ -195,7 +195,8 @@ extern void SSUpgradeFileBeforeCommit(); #define OPT_TRUNC_NUMERIC_TAIL_ZERO 16777216 #define OPT_ALLOW_ORDERBY_UNDISTINCT_COLUMN 33554432 #define OPT_SELECT_INTO_RETURN_NULL 67108864 -#define OPT_MAX 27 +#define OPT_ACCEPT_EMPTY_STR 134217728 +#define OPT_MAX 28 #define PLPSQL_OPT_FOR_LOOP 1 #define PLPSQL_OPT_OUTPARAM 2 @@ -209,6 +210,7 @@ extern void SSUpgradeFileBeforeCommit(); #define RETURN_NS (u_sess->utils_cxt.behavior_compat_flags & OPT_RETURN_NS_OR_NULL) #define CORRECT_TO_NUMBER (u_sess->utils_cxt.behavior_compat_flags & OPT_CORRECT_TO_NUMBER) #define SUPPORT_BIND_SEARCHPATH (u_sess->utils_cxt.behavior_compat_flags & OPT_BIND_SEARCHPATH) +#define ACCEPT_EMPTY_STR (u_sess->utils_cxt.behavior_compat_flags & OPT_ACCEPT_EMPTY_STR) /*CONCAT_VARIADIC controls 1.the variadic type process, and 2. td mode null return process in concat. By default, the * option is blank and the behavior is new and compatible with current A and C mode, if the option is set, the * behavior is old and the same as previous GAUSSDB kernel. */ diff --git a/src/test/regress/expected/accept_empty_str.out b/src/test/regress/expected/accept_empty_str.out new file mode 100644 index 000000000..e3eff0d93 --- /dev/null +++ b/src/test/regress/expected/accept_empty_str.out @@ -0,0 +1,1815 @@ +-- test about issue +create schema accept_schema; +set current_schema to 'accept_schema'; +create table tchar(c char(10)); +set behavior_compat_options to 'accept_empty_str'; +insert into tchar values(' '); +select * from tchar where c = ''; + c +------------ + +(1 row) + +select * from tchar where c = ' '; + c +------------ + +(1 row) + +select * from tchar where c is null; + c +--- +(0 rows) + +drop table tchar; +-- test about const str +select '' is null; + ?column? +---------- + f +(1 row) + +select ' ' is null; + ?column? +---------- + f +(1 row) + +select ' abc ' is null; + ?column? +---------- + f +(1 row) + +select length(''); + length +-------- + 0 +(1 row) + +select length(null); + length +-------- + +(1 row) + +select length(' '); + length +-------- + 1 +(1 row) + +select length(' abc '); + length +-------- + 5 +(1 row) + +select '123'::char(3) is null; + ?column? +---------- + f +(1 row) + +select ''::char(3) is null; + ?column? +---------- + f +(1 row) + +select '123'::varchar(3) is null; + ?column? +---------- + f +(1 row) + +select ''::varchar(3) is null; + ?column? +---------- + f +(1 row) + +select '123'::text is null; + ?column? +---------- + f +(1 row) + +select ''::text is null; + ?column? +---------- + f +(1 row) + +select '123'::clob is null; + ?column? +---------- + f +(1 row) + +select ''::clob is null; + ?column? +---------- + f +(1 row) + +select '123'::blob is null; + ?column? +---------- + f +(1 row) + +select ''::blob is null; + ?column? +---------- + f +(1 row) + +select '123'::bytea is null; + ?column? +---------- + f +(1 row) + +select ''::bytea is null; + ?column? +---------- + f +(1 row) + +select '123'::int1 is null; + ?column? +---------- + f +(1 row) + +select ''::int1 is null; +ERROR: invalid input syntax for integer: "" +LINE 1: select ''::int1 is null; + ^ +select '123'::int2 is null; + ?column? +---------- + f +(1 row) + +select ''::int2 is null; +ERROR: invalid input syntax for integer: "" +LINE 1: select ''::int2 is null; + ^ +select '123'::int is null; + ?column? +---------- + f +(1 row) + +select ''::int is null; +ERROR: invalid input syntax for integer: "" +LINE 1: select ''::int is null; + ^ +select '123'::int8 is null; + ?column? +---------- + f +(1 row) + +select ''::int8 is null; +ERROR: invalid input syntax for type bigint: "" +LINE 1: select ''::int8 is null; + ^ +select '123'::float4 is null; + ?column? +---------- + f +(1 row) + +select ''::float4 is null; +ERROR: invalid input syntax for type real: "" +LINE 1: select ''::float4 is null; + ^ +select '123'::float8 is null; + ?column? +---------- + f +(1 row) + +select ''::float8 is null; +ERROR: invalid input syntax for type double precision: "" +LINE 1: select ''::float8 is null; + ^ +select '123'::numeric is null; + ?column? +---------- + f +(1 row) + +select ''::numeric is null; +ERROR: invalid input syntax for type numeric: "" +LINE 1: select ''::numeric is null; + ^ +select ''::date is null; +ERROR: invalid input syntax for type timestamp: "" +LINE 1: select ''::date is null; + ^ +select ''::time is null; +ERROR: invalid input syntax for type time: "" +LINE 1: select ''::time is null; + ^ +select ''::timestamp is null; +ERROR: invalid input syntax for type timestamp: "" +LINE 1: select ''::timestamp is null; + ^ +-- test about var str +create table result_tab ("statement" text, result text); +declare +str_empty text := ''; +str_space text := ' '; +str_num text := '123'; +str text := ' abc '; +begin + insert into result_tab select 'select str_empty is null', str_empty is null; + insert into result_tab select 'select str_space is null', str_space is null; + insert into result_tab select 'select str is null', str is null; + insert into result_tab select 'select length(str_empty)', length(str_empty); + insert into result_tab select 'select length(null)', length(null); + insert into result_tab select 'select length(str_space)', length(str_space); + insert into result_tab select 'select length(str)', length(str); + insert into result_tab select 'select str_num::text is null', str_num::text is null; + insert into result_tab select 'select str_empty::text is null', str_empty::text is null; + insert into result_tab select 'select str_num::bytea is null;', str_num::bytea is null; + insert into result_tab select 'select str_empty::bytea is null', str_empty::bytea is null; + insert into result_tab select 'select str_num::int is null', str_num::int is null; + insert into result_tab select 'select str_num::float8 is null', str_num::float8 is null; + insert into result_tab select 'select str_num::numeric is null', str_num::numeric is null; +end; +/ +select * from result_tab; + statement | result +---------------------------------+-------- + select str_empty is null | false + select str_space is null | false + select str is null | false + select length(str_empty) | 0 + select length(null) | + select length(str_space) | 1 + select length(str) | 5 + select str_num::text is null | false + select str_empty::text is null | false + select str_num::bytea is null; | false + select str_empty::bytea is null | false + select str_num::int is null | false + select str_num::float8 is null | false + select str_num::numeric is null | false +(14 rows) + +-- test about function which return str +SELECT overlay('hello' placing 'world' from 2 for 3 ) is null; + ?column? +---------- + f +(1 row) + +SELECT overlay('hello' placing '' from 1 for 5 ) is null; + ?column? +---------- + f +(1 row) + +SELECT quote_ident('') is null; + ?column? +---------- + f +(1 row) + +SELECT quote_literal('') is null; + ?column? +---------- + f +(1 row) + +SELECT quote_nullable('') is null; + ?column? +---------- + f +(1 row) + +SELECT reverse('') is null; + ?column? +---------- + f +(1 row) + +SELECT ''||'' is null; + ?column? +---------- + false +(1 row) + +SELECT ''||41; + ?column? +---------- + 41 +(1 row) + +SELECT lower('') is null; + ?column? +---------- + f +(1 row) + +SELECT initcap('') is null; + ?column? +---------- + f +(1 row) + +SELECT ascii(''); + ascii +------- + 0 +(1 row) + +SELECT lpad('yes', 5) is null; + ?column? +---------- + f +(1 row) + +SELECT lpad('yes', 1) is null; + ?column? +---------- + f +(1 row) + +SELECT lpad('yes', 0) is null; + ?column? +---------- + f +(1 row) + +SELECT lpad('yes', 5, 'z') is null; + ?column? +---------- + f +(1 row) + +SELECT lpad('yes', 1, 'z') is null; + ?column? +---------- + f +(1 row) + +SELECT lpad('yes', 0, 'z') is null; + ?column? +---------- + f +(1 row) + +SELECT rpad('yes', 5) is null; + ?column? +---------- + f +(1 row) + +SELECT rpad('yes', 1) is null; + ?column? +---------- + f +(1 row) + +SELECT rpad('yes', 0) is null; + ?column? +---------- + f +(1 row) + +SELECT rpad('yes', 5, 'z') is null; + ?column? +---------- + f +(1 row) + +SELECT rpad('yes', 1, 'z') is null; + ?column? +---------- + f +(1 row) + +SELECT rpad('yes', 0, 'z') is null; + ?column? +---------- + f +(1 row) + +SELECT btrim('yzy', 'y') is null; + ?column? +---------- + f +(1 row) + +SELECT btrim('zzz', 'z') is null; + ?column? +---------- + f +(1 row) + +SELECT ltrim('yzy', 'y') is null; + ?column? +---------- + f +(1 row) + +SELECT ltrim('zzz', 'z') is null; + ?column? +---------- + f +(1 row) + +SELECT rtrim('yzy', 'y') is null; + ?column? +---------- + f +(1 row) + +SELECT rtrim('zzz', 'z') is null; + ?column? +---------- + f +(1 row) + +SELECT btrim(' z ') is null; + ?column? +---------- + f +(1 row) + +SELECT btrim(' ') is null; + ?column? +---------- + f +(1 row) + +SELECT ltrim(' z ') is null; + ?column? +---------- + f +(1 row) + +SELECT ltrim(' ') is null; + ?column? +---------- + f +(1 row) + +SELECT rtrim(' z ') is null; + ?column? +---------- + f +(1 row) + +SELECT rtrim(' ') is null; + ?column? +---------- + f +(1 row) + +SELECT translate('xyx', 'x', 'z') is null; + ?column? +---------- + f +(1 row) + +SELECT translate('xzx', 'x', '') is null; + ?column? +---------- + f +(1 row) + +SELECT translate('xxx', 'x', '') is null; + ?column? +---------- + f +(1 row) + +SELECT translate('xxx', 'x', ' ') is null; + ?column? +---------- + f +(1 row) + +SELECT translate('xxx', '', 'z') is null; + ?column? +---------- + f +(1 row) + +SELECT translate('', 'x', 'z') is null; + ?column? +---------- + f +(1 row) + +SELECT repeat('a', 3) is null; + ?column? +---------- + f +(1 row) + +SELECT repeat('a', 0) is null; + ?column? +---------- + f +(1 row) + +SELECT repeat(' ', 3) is null; + ?column? +---------- + f +(1 row) + +SELECT repeat(' ', 0) is null; + ?column? +---------- + f +(1 row) + +SELECT repeat('', 3) is null; + ?column? +---------- + f +(1 row) + +SELECT repeat('', 0) is null; + ?column? +---------- + f +(1 row) + +SELECT pg_catalog.oidvectortypes('123 456') is null; + ?column? +---------- + f +(1 row) + +SELECT pg_catalog.oidvectortypes('') is null; + ?column? +---------- + f +(1 row) + +SELECT pg_catalog.oidvectortypes(' ') is null; + ?column? +---------- + f +(1 row) + +SELECT regexp_replace('Thomas', '.[mN]a.', 'M') is null; + ?column? +---------- + f +(1 row) + +SELECT regexp_replace('omas', '.[mN]a.', ''); + regexp_replace +---------------- + +(1 row) + +SELECT regexp_replace('Thomas', '', 'M') is null; + ?column? +---------- + f +(1 row) + +SELECT regexp_replace('', '.[mN]a.', 'M') is null; + ?column? +---------- + f +(1 row) + +SELECT regexp_replace('omas', '.[mN]a.', '') is null; + ?column? +---------- + f +(1 row) + +SELECT regexp_replace('foobarbaz','b(..)', E'X\\1Y', 2, 2, 'n') is null; + ?column? +---------- + f +(1 row) + +SELECT regexp_replace('bar','b(..)', '', 1, 1, 'n') is null; + ?column? +---------- + f +(1 row) + +SELECT regexp_replace('foobarbaz','b(..)', E'X\\1Y', 'g') is null; + ?column? +---------- + f +(1 row) + +SELECT regexp_replace('abc','abc', '', 'g') is null; + ?column? +---------- + f +(1 row) + +SELECT regexp_substr('str','st') is null; + ?column? +---------- + f +(1 row) + +SELECT regexp_substr('str','[ac]') is null; + ?column? +---------- + t +(1 row) + +SELECT regexp_substr('str','') is null; + ?column? +---------- + f +(1 row) + +SELECT regexp_substr('','st') is null; + ?column? +---------- + t +(1 row) + +SELECT regexp_split_to_table('hello world', E'\\s+') is null; + ?column? +---------- + f + f +(2 rows) + +SELECT regexp_split_to_table('', E'\\s+') is null; + ?column? +---------- + f +(1 row) + +SELECT regexp_split_to_table('hello world', '') is null; + ?column? +---------- + f + f + f + f + f + f + f + f + f + f + f +(11 rows) + +SELECT regexp_split_to_table('hello world', null) is null; + ?column? +---------- +(0 rows) + +SELECT substr('123', 3) is null; + ?column? +---------- + f +(1 row) + +SELECT substr('123', 4) is null; + ?column? +---------- + f +(1 row) + +SELECT substr('123', 1, 2) is null; + ?column? +---------- + f +(1 row) + +SELECT substr('123', 1, 0) is null; + ?column? +---------- + f +(1 row) + +SELECT substr('123', 1, -1) is null; + ?column? +---------- + f +(1 row) + +SELECT substr('123'::bytea, 3) is null; + ?column? +---------- + f +(1 row) + +SELECT substr('123'::bytea, 4) is null; + ?column? +---------- + f +(1 row) + +SELECT substr('123'::bytea, 1, 2) is null; + ?column? +---------- + f +(1 row) + +SELECT substr('123'::bytea, 1, 0) is null; + ?column? +---------- + f +(1 row) + +SELECT substr('123'::bytea, 1, -1) is null; + ?column? +---------- + f +(1 row) + +SELECT substrb('123', 3) is null; + ?column? +---------- + f +(1 row) + +SELECT substrb('123', 4) is null; + ?column? +---------- + f +(1 row) + +SELECT substrb('123', 1, 2) is null; + ?column? +---------- + f +(1 row) + +SELECT substrb('123', 1, 0) is null; + ?column? +---------- + f +(1 row) + +SELECT substrb('123', 1, -1) is null; + ?column? +---------- + f +(1 row) + +SELECT substring('123', 3) is null; + ?column? +---------- + f +(1 row) + +SELECT substring('123', 4) is null; + ?column? +---------- + f +(1 row) + +SELECT substring('123', 1, 2) is null; + ?column? +---------- + f +(1 row) + +SELECT substring('123', 1, 0) is null; + ?column? +---------- + f +(1 row) + +SELECT substring('123', 1, -1) is null; +ERROR: negative substring length not allowed +SELECT substring('123'::bytea, 3) is null; + ?column? +---------- + f +(1 row) + +SELECT substring('123'::bytea, 4) is null; + ?column? +---------- + f +(1 row) + +SELECT substring('123'::bytea, 1, 2) is null; + ?column? +---------- + f +(1 row) + +SELECT substring('123'::bytea, 1, 0) is null; + ?column? +---------- + f +(1 row) + +SELECT substring('123'::bytea, 1, -1) is null; +ERROR: negative substring length not allowed +SELECT replace('abc', 'ab', 'd') is null; + ?column? +---------- + f +(1 row) + +SELECT replace('abc', 'abc', '') is null; + ?column? +---------- + f +(1 row) + +SELECT replace('abc', 'abc', null) is null; + ?column? +---------- + f +(1 row) + +SELECT replace('abc', 'ab', '') is null; + ?column? +---------- + f +(1 row) + +SELECT replace('abc', 'ab', null) is null; + ?column? +---------- + f +(1 row) + +SELECT replace('abc', '', 'd') is null; + ?column? +---------- + f +(1 row) + +SELECT replace('abc', null, 'd') is null; + ?column? +---------- + f +(1 row) + +SELECT replace('', 'ab', 'd') is null; + ?column? +---------- + f +(1 row) + +SELECT replace(null, 'ab', 'd') is null; + ?column? +---------- + t +(1 row) + +SELECT replace('abc', 'ab') is null; + ?column? +---------- + f +(1 row) + +SELECT replace('abc', 'abc') is null; + ?column? +---------- + f +(1 row) + +SELECT replace('abc', '') is null; + ?column? +---------- + f +(1 row) + +SELECT replace('abc', null) is null; + ?column? +---------- + f +(1 row) + +SELECT replace('', 'ab') is null; + ?column? +---------- + f +(1 row) + +SELECT replace(null, 'ab') is null; + ?column? +---------- + t +(1 row) + +SELECT split_part('1~2~3', '~', 3) is null; + ?column? +---------- + f +(1 row) + +SELECT split_part('1~2~3~', '~', 4) is null; + ?column? +---------- + f +(1 row) + +SELECT split_part('1~2~3', '~', -1) is null; +ERROR: field position must be greater than zero +SELECT split_part('1~2~3', '', 1) is null; + ?column? +---------- + f +(1 row) + +SELECT split_part('1~2~3', '', 2) is null; + ?column? +---------- + f +(1 row) + +SELECT split_part('1~2~3', null, 1) is null; + ?column? +---------- + t +(1 row) + +SELECT split_part('', '~', 1) is null; + ?column? +---------- + f +(1 row) + +SELECT split_part(null, '~', 1) is null; + ?column? +---------- + t +(1 row) + +SELECT split_part('1~2~3', '|', 1) is null; + ?column? +---------- + f +(1 row) + +SELECT split_part('1~2~3', '|', 2) is null; + ?column? +---------- + f +(1 row) + +SELECT concat('Hello', ' World!') is null; + ?column? +---------- + f +(1 row) + +SELECT concat('', ' World!') is null; + ?column? +---------- + f +(1 row) + +SELECT concat('Hello', '') is null; + ?column? +---------- + f +(1 row) + +SELECT concat('', '') is null; + ?column? +---------- + f +(1 row) + +SELECT concat('Hello', null) is null; + ?column? +---------- + f +(1 row) + +SELECT concat(null, ' World!') is null; + ?column? +---------- + f +(1 row) + +SELECT concat_ws(',', 'ABCDE', 2, NULL, 22); + concat_ws +------------ + ABCDE,2,22 +(1 row) + +SELECT concat_ws('', 'ABCDE', 2, NULL, 22); + concat_ws +----------- + ABCDE222 +(1 row) + +SELECT concat_ws(',', '') is null; + ?column? +---------- + f +(1 row) + +SELECT concat_ws('', '') is null; + ?column? +---------- + f +(1 row) + +SELECT left('abcde', 2) is null; + ?column? +---------- + f +(1 row) + +SELECT left('abcde', 0) is null; + ?column? +---------- + f +(1 row) + +SELECT left('abcde', -1) is null; + ?column? +---------- + f +(1 row) + +SELECT left('', 1) is null; + ?column? +---------- + f +(1 row) + +SELECT left('', 0) is null; + ?column? +---------- + f +(1 row) + +SELECT left('', -1) is null; + ?column? +---------- + f +(1 row) + +SELECT right('abcde', 2) is null; + ?column? +---------- + f +(1 row) + +SELECT right('abcde', 0) is null; + ?column? +---------- + f +(1 row) + +SELECT right('abcde', -1) is null; + ?column? +---------- + f +(1 row) + +SELECT right('', 1) is null; + ?column? +---------- + f +(1 row) + +SELECT right('', 0) is null; + ?column? +---------- + f +(1 row) + +SELECT right('', -1) is null; + ?column? +---------- + f +(1 row) + +SELECT format('Hello %s', 'World'); + format +------------- + Hello World +(1 row) + +SELECT format('Hello %s', ''); + format +-------- + Hello +(1 row) + +SELECT format('%s', 'World'); + format +-------- + World +(1 row) + +SELECT format('', 'World'); + format +-------- + +(1 row) + +SELECT format('', ''); + format +-------- + +(1 row) + +SELECT array_to_string(ARRAY[1, 2, 3, NULL, 5], ',') AS RESULT; + result +--------- + 1,2,3,5 +(1 row) + +SELECT array_to_string(ARRAY[1, 2, 3, NULL, 5], '') AS RESULT; + result +-------- + 1235 +(1 row) + +SELECT array_to_string(ARRAY[1, 2, 3, NULL, 5], null) AS RESULT; + result +-------- + +(1 row) + +SELECT array_to_string(ARRAY[NULL], '') AS RESULT; + result +-------- + +(1 row) + +SELECT array_to_string(ARRAY[NULL], null) AS RESULT; + result +-------- + +(1 row) + +SELECT array_to_string(ARRAY[1, 2, 3, NULL, 5], ',', '*') AS RESULT; + result +----------- + 1,2,3,*,5 +(1 row) + +SELECT array_to_string(ARRAY[1, 2, 3, NULL, 5], '', '') AS RESULT; + result +-------- + 1235 +(1 row) + +SELECT array_to_string(ARRAY[1, 2, 3, NULL, 5], null, null) AS RESULT; + result +-------- + +(1 row) + +SELECT array_to_string(ARRAY[NULL], '', '') AS RESULT; + result +-------- + +(1 row) + +SELECT array_to_string(ARRAY[NULL], null, null) AS RESULT; + result +-------- + +(1 row) + +SELECT nlssort('A', 'nls_sort=schinese_pinyin_m') is null; + ?column? +---------- + f +(1 row) + +SELECT nlssort('', 'nls_sort=schinese_pinyin_m') is null; + ?column? +---------- + f +(1 row) + +SELECT nlssort('A', '') is null; +ERROR: Sort method is not supported! +DETAIL: Not support the given sort method. +SELECT convert('text_in_utf8'::bytea, 'UTF8', 'GBK') is null; + ?column? +---------- + f +(1 row) + +SELECT convert(''::bytea, 'UTF8', 'GBK') is null; + ?column? +---------- + f +(1 row) + +SELECT convert('text_in_utf8'::bytea, '', 'GBK') is null; +ERROR: invalid source encoding name "" +SELECT convert('text_in_utf8'::bytea, 'UTF8', '') is null; +ERROR: invalid destination encoding name "" +SELECT convert_from('text_in_utf8'::bytea, 'UTF8') is null; + ?column? +---------- + f +(1 row) + +SELECT convert_from(''::bytea, 'UTF8') is null; + ?column? +---------- + f +(1 row) + +SELECT convert_from('text_in_utf8'::bytea, '') is null; +ERROR: invalid source encoding name "" +SELECT convert_to('text_in_utf8'::bytea, 'UTF8') is null; +ERROR: function convert_to(bytea, unknown) does not exist +LINE 1: SELECT convert_to('text_in_utf8'::bytea, 'UTF8') is null; + ^ +HINT: No function matches the given name and argument types. You might need to add explicit type casts. +SELECT convert_to(''::bytea, 'UTF8') is null; +ERROR: function convert_to(bytea, unknown) does not exist +LINE 1: SELECT convert_to(''::bytea, 'UTF8') is null; + ^ +HINT: No function matches the given name and argument types. You might need to add explicit type casts. +SELECT convert_to('text_in_utf8'::bytea, '') is null; +ERROR: function convert_to(bytea, unknown) does not exist +LINE 1: SELECT convert_to('text_in_utf8'::bytea, '') is null; + ^ +HINT: No function matches the given name and argument types. You might need to add explicit type casts. +SELECT md5('ABC') is null; + ?column? +---------- + f +(1 row) + +SELECT md5('') is null; + ?column? +---------- + f +(1 row) + +SELECT sha('ABC') is null; +ERROR: sha/sha1 is supported only in B-format database +SELECT sha('') is null; +ERROR: sha/sha1 is supported only in B-format database +SELECT sha1('ABC') is null; +ERROR: sha/sha1 is supported only in B-format database +SELECT sha1('') is null; +ERROR: sha/sha1 is supported only in B-format database +SELECT sha2('ABC') is null; +ERROR: function sha2(unknown) does not exist +LINE 1: SELECT sha2('ABC') is null; + ^ +HINT: No function matches the given name and argument types. You might need to add explicit type casts. +SELECT sha2('') is null; +ERROR: function sha2(unknown) does not exist +LINE 1: SELECT sha2('') is null; + ^ +HINT: No function matches the given name and argument types. You might need to add explicit type casts. +SELECT decode('MTIzAAE=', 'base64') is null; + ?column? +---------- + f +(1 row) + +SELECT decode('', 'base64') is null; + ?column? +---------- + f +(1 row) + +SELECT decode('MTIzAAE=', '') is null; +ERROR: unrecognized encoding: "" +select similar_escape('\s+ab','2') is null; + ?column? +---------- + f +(1 row) + +select similar_escape('\s+ab','') is null; + ?column? +---------- + f +(1 row) + +select similar_escape('','2') is null; + ?column? +---------- + f +(1 row) + +select svals('"aa"=>"bb"') is null; + ?column? +---------- + f +(1 row) + +select svals('') is null; + ?column? +---------- +(0 rows) + +select tconvert('aa', 'bb') is null; + ?column? +---------- + f +(1 row) + +select tconvert('', 'bb') is null; + ?column? +---------- + f +(1 row) + +select tconvert('aa', '') is null; + ?column? +---------- + f +(1 row) + +select tconvert('', '') is null; + ?column? +---------- + f +(1 row) + +SELECT encode(E'123\\000\\001', 'base64') is null; + ?column? +---------- + f +(1 row) + +SELECT encode('', 'base64') is null; + ?column? +---------- + f +(1 row) + +SELECT encode(E'123\\000\\001', '') is null; +ERROR: unrecognized encoding: "" +-- test about vec +CREATE TABLE vec_t1 +( + c varchar +) WITH (ORIENTATION = COLUMN); +CREATE TABLE vec_t2 +( + c varchar not null +) WITH (ORIENTATION = COLUMN); +insert into vec_t1 values(''); +insert into vec_t1 values(' '); +insert into vec_t1 values('abc'); +insert into vec_t2 values(''); +insert into vec_t2 values(' '); +insert into vec_t2 values('abc'); +select c as input, substr(c, 1, 2) as result, result is null as is_null from vec_t1; + input | result | is_null +-------+--------+--------- + | | f + | | f + abc | ab | f +(3 rows) + +select c as input, substr(c, 1, 1) as result, result is null as is_null from vec_t1; + input | result | is_null +-------+--------+--------- + | | f + | | f + abc | a | f +(3 rows) + +select c as input, substr(c, 0, 2) as result, result is null as is_null from vec_t1; + input | result | is_null +-------+--------+--------- + | | f + | | f + abc | ab | f +(3 rows) + +select c as input, substr(c, 1, 0) as result, result is null as is_null from vec_t1; + input | result | is_null +-------+--------+--------- + | | f + | | f + abc | | f +(3 rows) + +select c as input, substr(c, 1, -1) as result, result is null as is_null from vec_t1; + input | result | is_null +-------+--------+--------- + | | f + | | f + abc | | f +(3 rows) + +delete from vec_t1; +insert into vec_t1 values(''); +insert into vec_t1 values('yzy'); +insert into vec_t1 values('zzz'); +insert into vec_t1 values(' z '); +insert into vec_t1 values(' '); +SELECT c as input, btrim(c, 'y') as result, result is null as isnull from vec_t1; + input | result | isnull +-------+--------+-------- + | | f + yzy | z | f + zzz | zzz | f + z | z | f + | | f +(5 rows) + +SELECT c as input, btrim(c, 'z') as result, result is null as isnull from vec_t1; + input | result | isnull +-------+--------+-------- + | | f + yzy | yzy | f + zzz | | f + z | z | f + | | f +(5 rows) + +SELECT c as input, btrim(c) as result, result is null as isnull from vec_t1; + input | result | isnull +-------+--------+-------- + | | f + yzy | yzy | f + zzz | zzz | f + z | z | f + | | f +(5 rows) + +SELECT c as input, rtrim(c, 'y') as result, result is null as isnull from vec_t1; + input | result | isnull +-------+--------+-------- + | | f + yzy | yz | f + zzz | zzz | f + z | z | f + | | f +(5 rows) + +SELECT c as input, rtrim(c, 'z') as result, result is null as isnull from vec_t1; + input | result | isnull +-------+--------+-------- + | | f + yzy | yzy | f + zzz | | f + z | z | f + | | f +(5 rows) + +SELECT c as input, rtrim(c) as result, result is null as isnull from vec_t1; + input | result | isnull +-------+--------+-------- + | | f + yzy | yzy | f + zzz | zzz | f + z | z | f + | | f +(5 rows) + +-- test row to vec +set try_vector_engine_strategy=force; +create table vec_t3(c text); +insert into vec_t3 values(''); +insert into vec_t3 values(' '); +insert into vec_t3 values('abc'); +explain analyze select c as input, substr(c, 1, 2) as result, result is null as is_null from vec_t3; +--?QUERY PLAN.* +--?.* +--?Row Adapter.* +--?Vector Adapter(type: BATCH MODE).* +--?Seq Scan on vec_t3.* +--?.* +(4 rows) + +select c as input, substr(c, 1, 2) as result, result is null as is_null from vec_t3; + input | result | is_null +-------+--------+--------- + | | f + | | f + abc | ab | f +(3 rows) + +select c as input, substr(c, 1, 1) as result, result is null as is_null from vec_t3; + input | result | is_null +-------+--------+--------- + | | f + | | f + abc | a | f +(3 rows) + +select c as input, substr(c, 0, 2) as result, result is null as is_null from vec_t3; + input | result | is_null +-------+--------+--------- + | | f + | | f + abc | ab | f +(3 rows) + +select c as input, substr(c, 1, 0) as result, result is null as is_null from vec_t3; + input | result | is_null +-------+--------+--------- + | | f + | | f + abc | | f +(3 rows) + +delete from vec_t3; +insert into vec_t3 values(''); +insert into vec_t3 values('yzy'); +insert into vec_t3 values('zzz'); +insert into vec_t3 values(' z '); +insert into vec_t3 values(' '); +explain analyze SELECT c as input, rtrim(c, 'y') as result, result is null as isnull from vec_t3; +--?QUERY PLAN.* +--?.* +--?Row Adapter.* +--?Vector Adapter(type: BATCH MODE).* +--?Seq Scan on vec_t3.* +--?.* +(4 rows) + +SELECT c as input, rtrim(c, 'y') as result, result is null as isnull from vec_t3; + input | result | isnull +-------+--------+-------- + | | f + yzy | yz | f + zzz | zzz | f + z | z | f + | | f +(5 rows) + +SELECT c as input, rtrim(c, 'z') as result, result is null as isnull from vec_t3; + input | result | isnull +-------+--------+-------- + | | f + yzy | yzy | f + zzz | | f + z | z | f + | | f +(5 rows) + +SELECT c as input, rtrim(c) as result, result is null as isnull from vec_t3; + input | result | isnull +-------+--------+-------- + | | f + yzy | yzy | f + zzz | zzz | f + z | z | f + | | f +(5 rows) + +create table pad2_tab(a text, b int); +insert into pad2_tab values('yes', 5), ('yes', 1), ('yes', 0); +create table pad3_tab(a text, b int, c text); +insert into pad3_tab values('yes', 5, 'z'), ('yes', 1, 'z'), ('yes', 0, 'z'); +SELECT a as p1, b as p2, lpad(a, b) is null from pad2_tab; + p1 | p2 | ?column? +-----+----+---------- + yes | 5 | f + yes | 1 | f + yes | 0 | f +(3 rows) + +SELECT a as p1, b as p2, lpad(a, b, c) is null from pad3_tab; + p1 | p2 | ?column? +-----+----+---------- + yes | 5 | f + yes | 1 | f + yes | 0 | f +(3 rows) + +SELECT a as p1, b as p2, rpad(a, b) is null from pad2_tab; + p1 | p2 | ?column? +-----+----+---------- + yes | 5 | f + yes | 1 | f + yes | 0 | f +(3 rows) + +SELECT a as p1, b as p2, rpad(a, b, c) is null from pad3_tab; + p1 | p2 | ?column? +-----+----+---------- + yes | 5 | f + yes | 1 | f + yes | 0 | f +(3 rows) + +create table trim2(a text, b text); +insert into trim2 values('yzy', 'y'), ('zzz', 'z'); +create table trim1(a text); +insert into trim1 values(' z '), (' '); +SELECT a as p1, b as p2, btrim(a, b) is null from trim2; + p1 | p2 | ?column? +-----+----+---------- + yzy | y | f + zzz | z | f +(2 rows) + +SELECT a as p1, b as p2, ltrim(a, b) is null from trim2; + p1 | p2 | ?column? +-----+----+---------- + yzy | y | f + zzz | z | f +(2 rows) + +SELECT a as p1, b as p2, rtrim(a, b) is null from trim2; + p1 | p2 | ?column? +-----+----+---------- + yzy | y | f + zzz | z | f +(2 rows) + +SELECT a as p1, btrim(a) is null from trim1; + p1 | ?column? +-----+---------- + z | f + | f +(2 rows) + +SELECT a as p1, ltrim(a) is null from trim1; + p1 | ?column? +-----+---------- + z | f + | f +(2 rows) + +SELECT a as p1, rtrim(a) is null from trim1; + p1 | ?column? +-----+---------- + z | f + | f +(2 rows) + +create table translate3(a text, b text, c text); +insert into translate3 values('xyx', 'x', 'z'), ('xzx', 'x', ''), ('xxx', 'x', ''), ('xxx', 'x', ' '), ('xxx', '', 'z'), ('', 'x', 'z'); +SELECT a as p1, b as p2, c as p3, translate(a, b, c) is null from translate3; + p1 | p2 | p3 | ?column? +-----+----+----+---------- + xyx | x | z | f + xzx | x | | f + xxx | x | | f + xxx | x | | f + xxx | | z | f + | x | z | f +(6 rows) + +create table repeat2 (a text, b int); +insert into repeat2 values('a', 3), ('a', 0), (' ', 3), ('', 3), ('', 0); +SELECT a as p1, b as p2, repeat(a, b) is null from repeat2; + p1 | p2 | ?column? +----+----+---------- + a | 3 | f + a | 0 | f + | 3 | f + | 3 | f + | 0 | f +(5 rows) + +create table oidvectortypes1 (a oidvector); +insert into oidvectortypes1 values ('123 456'), (''), (' '); +SELECT a as p1, oidvectortypes(a) is null from oidvectortypes1; + p1 | ?column? +---------+---------- + 123 456 | f + | f + | f +(3 rows) + +create table regexp_replace3 (a text, b text, c text); +insert into regexp_replace3 values('Thomas', '.[mN]a.', 'M'), ('omas', '.[mN]a.', ''), ('Thomas', '', 'M'), ('', '.[mN]a.', 'M'), ('omas', '.[mN]a.', ''); +SELECT a as p1, b as p2, c as p3, regexp_replace(a, b, c) is null from regexp_replace3; + p1 | p2 | p3 | ?column? +--------+---------+----+---------- + Thomas | .[mN]a. | M | f + omas | .[mN]a. | | f + Thomas | | M | f + | .[mN]a. | M | f + omas | .[mN]a. | | f +(5 rows) + +create table regexp_replace6 (a text, b text, c text, d int, e int, f text); +insert into regexp_replace6 values ('foobarbaz','b(..)', E'X\\1Y', 2, 2, 'n'), ('bar','b(..)', '', 1, 1, 'n'); +SELECT a as p1, b as p2, c as p3, d as p4, e as p5, f as p6, regexp_replace(a, b, c, d, e, f) is null from regexp_replace6; + p1 | p2 | p3 | p4 | p5 | p6 | ?column? +-----------+-------+------+----+----+----+---------- + foobarbaz | b(..) | X\1Y | 2 | 2 | n | f + bar | b(..) | | 1 | 1 | n | f +(2 rows) + +create table regexp_split_to_table2 (a text, b text); +insert into regexp_split_to_table2 values('hello world', E'\\s+'), ('', E'\\s+'), ('hello world', ''), ('hello world', null); +SELECT a as p1, b as p2, regexp_split_to_table(a, b) is null from regexp_split_to_table2; + p1 | p2 | ?column? +-------------+-----+---------- + hello world | \s+ | f + hello world | \s+ | f + | \s+ | f + hello world | | f + hello world | | f + hello world | | f + hello world | | f + hello world | | f + hello world | | f + hello world | | f + hello world | | f + hello world | | f + hello world | | f + hello world | | f +(14 rows) + +create table substr2(a bytea, b int); +insert into substr2 values ('123'::bytea, 3), ('123'::bytea, 4); +select a as p1, b as p2, substr(a, b) is null from substr2; + p1 | p2 | ?column? +----------+----+---------- + \x313233 | 3 | f + \x313233 | 4 | f +(2 rows) + +create table substr3(a bytea, b int, c int); +insert into substr3 values('123'::bytea, 1, 2), ('123'::bytea, 1, 0); +select a as p1, b as p2, c as p3, substr(a, b, c) is null from substr3; + p1 | p2 | p3 | ?column? +----------+----+----+---------- + \x313233 | 1 | 2 | f + \x313233 | 1 | 0 | f +(2 rows) + +create table replace3 (a text, b text, c text); +insert into replace3 values ('abc', 'ab', 'd'), ('abc', 'abc', ''), ('abc', 'abc', ''), ('abc', 'ab', ''), ('abc', 'ab', null), ('abc', '', 'd'), ('abc', null, 'd'), ('', 'ab', 'd'), (null, 'ab', 'd'); +SELECT a as p1, b as p2, c as p3, replace(a, b, c) is null from replace3; + p1 | p2 | p3 | ?column? +-----+-----+----+---------- + abc | ab | d | f + abc | abc | | f + abc | abc | | f + abc | ab | | f + abc | ab | | f + abc | | d | f + abc | | d | f + | ab | d | f + | ab | d | t +(9 rows) + +create table replace2 (a text, b text); +insert into replace2 values('abc', 'ab'), ('abc', 'abc'), ('abc', ''), ('abc', null), ('', 'ab'), (null, 'ab'); +SELECT a as p1, b as p2, replace(a, b) is null from replace2; + p1 | p2 | ?column? +-----+-----+---------- + abc | ab | f + abc | abc | f + abc | | f + abc | | f + | ab | f + | ab | t +(6 rows) + +create table split_part3 (a text, b text, c int); +insert into split_part3 values('1~2~3', '~', 3), ('1~2~3~', '~', 4), ('1~2~3', '', 1), ('1~2~3', null, 1), ('', '~', 1), (null, '~', 1); +SELECT a as p1, b as p2, c as p3, split_part(a, b, c) is null from split_part3; + p1 | p2 | p3 | ?column? +--------+----+----+---------- + 1~2~3 | ~ | 3 | f + 1~2~3~ | ~ | 4 | f + 1~2~3 | | 1 | f + 1~2~3 | | 1 | t + | ~ | 1 | f + | ~ | 1 | t +(6 rows) + +create table array_to_string2(a integer[], b text); +insert into array_to_string2 values(ARRAY[1, 2, 3, NULL, 5], ','), (ARRAY[1, 2, 3, NULL, 5], ''), (ARRAY[1, 2, 3, NULL, 5], null), (ARRAY[NULL], ''), (ARRAY[NULL], null); +SELECT a as p1, b as p2, array_to_string(a, b) AS RESULT from array_to_string2; + p1 | p2 | result +----------------+----+--------- + {1,2,3,NULL,5} | , | 1,2,3,5 + {1,2,3,NULL,5} | | 1235 + {1,2,3,NULL,5} | | + {NULL} | | + {NULL} | | +(5 rows) + +create table array_to_string3(a integer[], b text, c text); +insert into array_to_string3 values(ARRAY[1, 2, 3, NULL, 5], ',', '*'), (ARRAY[1, 2, 3, NULL, 5], '', ''), (ARRAY[1, 2, 3, NULL, 5], null, null), (ARRAY[NULL], '', ''), (ARRAY[NULL], null, null); +SELECT a as p1, b as p2, c as p3, array_to_string(a, b, c) AS RESULT from array_to_string3; + p1 | p2 | p3 | result +----------------+----+----+----------- + {1,2,3,NULL,5} | , | * | 1,2,3,*,5 + {1,2,3,NULL,5} | | | 1235 + {1,2,3,NULL,5} | | | + {NULL} | | | + {NULL} | | | +(5 rows) + +set try_vector_engine_strategy=off; diff --git a/src/test/regress/expected/not_accept_empty_str.out b/src/test/regress/expected/not_accept_empty_str.out new file mode 100644 index 000000000..a2149a902 --- /dev/null +++ b/src/test/regress/expected/not_accept_empty_str.out @@ -0,0 +1,1822 @@ +-- test about issue +create schema not_accept_schema; +set current_schema to 'not_accept_schema'; +create table tchar(c char(10)); +set behavior_compat_options to ''; +insert into tchar values(' '); +select * from tchar where c = ''; + c +--- +(0 rows) + +select * from tchar where c = ' '; + c +------------ + +(1 row) + +select * from tchar where c is null; + c +--- +(0 rows) + +drop table tchar; +-- test about const str +select '' is null; + ?column? +---------- + t +(1 row) + +select ' ' is null; + ?column? +---------- + f +(1 row) + +select ' abc ' is null; + ?column? +---------- + f +(1 row) + +select length(''); + length +-------- + +(1 row) + +select length(null); + length +-------- + +(1 row) + +select length(' '); + length +-------- + 1 +(1 row) + +select length(' abc '); + length +-------- + 5 +(1 row) + +select '123'::char(3) is null; + ?column? +---------- + f +(1 row) + +select ''::char(3) is null; + ?column? +---------- + t +(1 row) + +select '123'::varchar(3) is null; + ?column? +---------- + f +(1 row) + +select ''::varchar(3) is null; + ?column? +---------- + t +(1 row) + +select '123'::text is null; + ?column? +---------- + f +(1 row) + +select ''::text is null; + ?column? +---------- + t +(1 row) + +select '123'::clob is null; + ?column? +---------- + f +(1 row) + +select ''::clob is null; + ?column? +---------- + t +(1 row) + +select '123'::blob is null; + ?column? +---------- + f +(1 row) + +select ''::blob is null; + ?column? +---------- + t +(1 row) + +select '123'::bytea is null; + ?column? +---------- + f +(1 row) + +select ''::bytea is null; + ?column? +---------- + t +(1 row) + +select '123'::int1 is null; + ?column? +---------- + f +(1 row) + +select ''::int1 is null; + ?column? +---------- + t +(1 row) + +select '123'::int2 is null; + ?column? +---------- + f +(1 row) + +select ''::int2 is null; + ?column? +---------- + t +(1 row) + +select '123'::int is null; + ?column? +---------- + f +(1 row) + +select ''::int is null; + ?column? +---------- + t +(1 row) + +select '123'::int8 is null; + ?column? +---------- + f +(1 row) + +select ''::int8 is null; + ?column? +---------- + t +(1 row) + +select '123'::float4 is null; + ?column? +---------- + f +(1 row) + +select ''::float4 is null; + ?column? +---------- + t +(1 row) + +select '123'::float8 is null; + ?column? +---------- + f +(1 row) + +select ''::float8 is null; + ?column? +---------- + t +(1 row) + +select '123'::numeric is null; + ?column? +---------- + f +(1 row) + +select ''::numeric is null; + ?column? +---------- + t +(1 row) + +select ''::date is null; + ?column? +---------- + t +(1 row) + +select ''::time is null; + ?column? +---------- + t +(1 row) + +select ''::timestamp is null; + ?column? +---------- + t +(1 row) + +-- test about var str +create table result_tab ("statement" text, result text); +declare +str_empty text := ''; +str_space text := ' '; +str_num text := '123'; +str text := ' abc '; +begin + insert into result_tab select 'select str_empty is null', str_empty is null; + insert into result_tab select 'select str_space is null', str_space is null; + insert into result_tab select 'select str is null', str is null; + insert into result_tab select 'select length(str_empty)', length(str_empty); + insert into result_tab select 'select length(null)', length(null); + insert into result_tab select 'select length(str_space)', length(str_space); + insert into result_tab select 'select length(str)', length(str); + insert into result_tab select 'select str_num::text is null', str_num::text is null; + insert into result_tab select 'select str_empty::text is null', str_empty::text is null; + insert into result_tab select 'select str_num::bytea is null;', str_num::bytea is null; + insert into result_tab select 'select str_empty::bytea is null', str_empty::bytea is null; + insert into result_tab select 'select str_num::int is null', str_num::int is null; + insert into result_tab select 'select str_num::float8 is null', str_num::float8 is null; + insert into result_tab select 'select str_num::numeric is null', str_num::numeric is null; +end; +/ +select * from result_tab; + statement | result +---------------------------------+-------- + select str_empty is null | true + select str_space is null | false + select str is null | false + select length(str_empty) | + select length(null) | + select length(str_space) | 1 + select length(str) | 5 + select str_num::text is null | false + select str_empty::text is null | true + select str_num::bytea is null; | false + select str_empty::bytea is null | true + select str_num::int is null | false + select str_num::float8 is null | false + select str_num::numeric is null | false +(14 rows) + +-- test about function which return str +SELECT overlay('hello' placing 'world' from 2 for 3 ) is null; + ?column? +---------- + f +(1 row) + +SELECT overlay('hello' placing '' from 1 for 5 ) is null; + ?column? +---------- + t +(1 row) + +SELECT quote_ident('') is null; + ?column? +---------- + t +(1 row) + +SELECT quote_literal('') is null; + ?column? +---------- + t +(1 row) + +SELECT quote_nullable('') is null; + ?column? +---------- + f +(1 row) + +SELECT reverse('') is null; + ?column? +---------- + t +(1 row) + +SELECT ''||'' is null; + ?column? +---------- + true +(1 row) + +SELECT ''||41; + ?column? +---------- + 41 +(1 row) + +SELECT lower('') is null; + ?column? +---------- + t +(1 row) + +SELECT initcap('') is null; + ?column? +---------- + t +(1 row) + +SELECT ascii(''); + ascii +------- + +(1 row) + +SELECT lpad('yes', 5) is null; + ?column? +---------- + f +(1 row) + +SELECT lpad('yes', 1) is null; + ?column? +---------- + f +(1 row) + +SELECT lpad('yes', 0) is null; + ?column? +---------- + t +(1 row) + +SELECT lpad('yes', 5, 'z') is null; + ?column? +---------- + f +(1 row) + +SELECT lpad('yes', 1, 'z') is null; + ?column? +---------- + f +(1 row) + +SELECT lpad('yes', 0, 'z') is null; + ?column? +---------- + t +(1 row) + +SELECT rpad('yes', 5) is null; + ?column? +---------- + f +(1 row) + +SELECT rpad('yes', 1) is null; + ?column? +---------- + f +(1 row) + +SELECT rpad('yes', 0) is null; + ?column? +---------- + t +(1 row) + +SELECT rpad('yes', 5, 'z') is null; + ?column? +---------- + f +(1 row) + +SELECT rpad('yes', 1, 'z') is null; + ?column? +---------- + f +(1 row) + +SELECT rpad('yes', 0, 'z') is null; + ?column? +---------- + t +(1 row) + +SELECT btrim('yzy', 'y') is null; + ?column? +---------- + f +(1 row) + +SELECT btrim('zzz', 'z') is null; + ?column? +---------- + t +(1 row) + +SELECT ltrim('yzy', 'y') is null; + ?column? +---------- + f +(1 row) + +SELECT ltrim('zzz', 'z') is null; + ?column? +---------- + t +(1 row) + +SELECT rtrim('yzy', 'y') is null; + ?column? +---------- + f +(1 row) + +SELECT rtrim('zzz', 'z') is null; + ?column? +---------- + t +(1 row) + +SELECT btrim(' z ') is null; + ?column? +---------- + f +(1 row) + +SELECT btrim(' ') is null; + ?column? +---------- + t +(1 row) + +SELECT ltrim(' z ') is null; + ?column? +---------- + f +(1 row) + +SELECT ltrim(' ') is null; + ?column? +---------- + t +(1 row) + +SELECT rtrim(' z ') is null; + ?column? +---------- + f +(1 row) + +SELECT rtrim(' ') is null; + ?column? +---------- + t +(1 row) + +SELECT translate('xyx', 'x', 'z') is null; + ?column? +---------- + f +(1 row) + +SELECT translate('xzx', 'x', '') is null; + ?column? +---------- + t +(1 row) + +SELECT translate('xxx', 'x', '') is null; + ?column? +---------- + t +(1 row) + +SELECT translate('xxx', 'x', ' ') is null; + ?column? +---------- + f +(1 row) + +SELECT translate('xxx', '', 'z') is null; + ?column? +---------- + t +(1 row) + +SELECT translate('', 'x', 'z') is null; + ?column? +---------- + t +(1 row) + +SELECT repeat('a', 3) is null; + ?column? +---------- + f +(1 row) + +SELECT repeat('a', 0) is null; + ?column? +---------- + t +(1 row) + +SELECT repeat(' ', 3) is null; + ?column? +---------- + f +(1 row) + +SELECT repeat(' ', 0) is null; + ?column? +---------- + t +(1 row) + +SELECT repeat('', 3) is null; + ?column? +---------- + t +(1 row) + +SELECT repeat('', 0) is null; + ?column? +---------- + t +(1 row) + +SELECT pg_catalog.oidvectortypes('123 456') is null; + ?column? +---------- + f +(1 row) + +SELECT pg_catalog.oidvectortypes('') is null; + ?column? +---------- + t +(1 row) + +SELECT pg_catalog.oidvectortypes(' ') is null; + ?column? +---------- + t +(1 row) + +SELECT regexp_replace('Thomas', '.[mN]a.', 'M') is null; + ?column? +---------- + f +(1 row) + +SELECT regexp_replace('omas', '.[mN]a.', ''); + regexp_replace +---------------- + +(1 row) + +SELECT regexp_replace('Thomas', '', 'M') is null; + ?column? +---------- + f +(1 row) + +SELECT regexp_replace('', '.[mN]a.', 'M') is null; + ?column? +---------- + t +(1 row) + +SELECT regexp_replace('omas', '.[mN]a.', '') is null; + ?column? +---------- + t +(1 row) + +SELECT regexp_replace('foobarbaz','b(..)', E'X\\1Y', 2, 2, 'n') is null; + ?column? +---------- + f +(1 row) + +SELECT regexp_replace('bar','b(..)', '', 1, 1, 'n') is null; + ?column? +---------- + t +(1 row) + +SELECT regexp_replace('foobarbaz','b(..)', E'X\\1Y', 'g') is null; + ?column? +---------- + f +(1 row) + +SELECT regexp_replace('abc','abc', '', 'g') is null; + ?column? +---------- + t +(1 row) + +SELECT regexp_substr('str','st') is null; + ?column? +---------- + f +(1 row) + +SELECT regexp_substr('str','[ac]') is null; + ?column? +---------- + t +(1 row) + +SELECT regexp_substr('str','') is null; + ?column? +---------- + t +(1 row) + +SELECT regexp_substr('','st') is null; + ?column? +---------- + t +(1 row) + +SELECT regexp_split_to_table('hello world', E'\\s+') is null; + ?column? +---------- + f + f +(2 rows) + +SELECT regexp_split_to_table('', E'\\s+') is null; + ?column? +---------- +(0 rows) + +SELECT regexp_split_to_table('hello world', '') is null; + ?column? +---------- +(0 rows) + +SELECT regexp_split_to_table('hello world', null) is null; + ?column? +---------- +(0 rows) + +SELECT substr('123', 3) is null; + ?column? +---------- + f +(1 row) + +SELECT substr('123', 4) is null; + ?column? +---------- + t +(1 row) + +SELECT substr('123', 1, 2) is null; + ?column? +---------- + f +(1 row) + +SELECT substr('123', 1, 0) is null; + ?column? +---------- + t +(1 row) + +SELECT substr('123', 1, -1) is null; + ?column? +---------- + t +(1 row) + +SELECT substr('123'::bytea, 3) is null; + ?column? +---------- + f +(1 row) + +SELECT substr('123'::bytea, 4) is null; + ?column? +---------- + t +(1 row) + +SELECT substr('123'::bytea, 1, 2) is null; + ?column? +---------- + f +(1 row) + +SELECT substr('123'::bytea, 1, 0) is null; + ?column? +---------- + t +(1 row) + +SELECT substr('123'::bytea, 1, -1) is null; + ?column? +---------- + t +(1 row) + +SELECT substrb('123', 3) is null; + ?column? +---------- + f +(1 row) + +SELECT substrb('123', 4) is null; + ?column? +---------- + t +(1 row) + +SELECT substrb('123', 1, 2) is null; + ?column? +---------- + f +(1 row) + +SELECT substrb('123', 1, 0) is null; + ?column? +---------- + t +(1 row) + +SELECT substrb('123', 1, -1) is null; + ?column? +---------- + t +(1 row) + +SELECT substring('123', 3) is null; + ?column? +---------- + f +(1 row) + +SELECT substring('123', 4) is null; + ?column? +---------- + t +(1 row) + +SELECT substring('123', 1, 2) is null; + ?column? +---------- + f +(1 row) + +SELECT substring('123', 1, 0) is null; + ?column? +---------- + t +(1 row) + +SELECT substring('123', 1, -1) is null; +ERROR: negative substring length not allowed +SELECT substring('123'::bytea, 3) is null; + ?column? +---------- + f +(1 row) + +SELECT substring('123'::bytea, 4) is null; + ?column? +---------- + f +(1 row) + +SELECT substring('123'::bytea, 1, 2) is null; + ?column? +---------- + f +(1 row) + +SELECT substring('123'::bytea, 1, 0) is null; + ?column? +---------- + f +(1 row) + +SELECT substring('123'::bytea, 1, -1) is null; +ERROR: negative substring length not allowed +SELECT replace('abc', 'ab', 'd') is null; + ?column? +---------- + f +(1 row) + +SELECT replace('abc', 'abc', '') is null; + ?column? +---------- + t +(1 row) + +SELECT replace('abc', 'abc', null) is null; + ?column? +---------- + t +(1 row) + +SELECT replace('abc', 'ab', '') is null; + ?column? +---------- + f +(1 row) + +SELECT replace('abc', 'ab', null) is null; + ?column? +---------- + f +(1 row) + +SELECT replace('abc', '', 'd') is null; + ?column? +---------- + f +(1 row) + +SELECT replace('abc', null, 'd') is null; + ?column? +---------- + f +(1 row) + +SELECT replace('', 'ab', 'd') is null; + ?column? +---------- + t +(1 row) + +SELECT replace(null, 'ab', 'd') is null; + ?column? +---------- + t +(1 row) + +SELECT replace('abc', 'ab') is null; + ?column? +---------- + f +(1 row) + +SELECT replace('abc', 'abc') is null; +ERROR: function returned NULL +SELECT replace('abc', '') is null; + ?column? +---------- + f +(1 row) + +SELECT replace('abc', null) is null; + ?column? +---------- + f +(1 row) + +SELECT replace('', 'ab') is null; + ?column? +---------- + t +(1 row) + +SELECT replace(null, 'ab') is null; + ?column? +---------- + t +(1 row) + +SELECT split_part('1~2~3', '~', 3) is null; + ?column? +---------- + f +(1 row) + +SELECT split_part('1~2~3~', '~', 4) is null; + ?column? +---------- + t +(1 row) + +SELECT split_part('1~2~3', '~', -1) is null; +ERROR: field position must be greater than zero +SELECT split_part('1~2~3', '', 1) is null; + ?column? +---------- + t +(1 row) + +SELECT split_part('1~2~3', '', 2) is null; + ?column? +---------- + t +(1 row) + +SELECT split_part('1~2~3', null, 1) is null; + ?column? +---------- + t +(1 row) + +SELECT split_part('', '~', 1) is null; + ?column? +---------- + t +(1 row) + +SELECT split_part(null, '~', 1) is null; + ?column? +---------- + t +(1 row) + +SELECT split_part('1~2~3', '|', 1) is null; + ?column? +---------- + f +(1 row) + +SELECT split_part('1~2~3', '|', 2) is null; + ?column? +---------- + t +(1 row) + +SELECT concat('Hello', ' World!') is null; + ?column? +---------- + f +(1 row) + +SELECT concat('', ' World!') is null; + ?column? +---------- + f +(1 row) + +SELECT concat('Hello', '') is null; + ?column? +---------- + f +(1 row) + +SELECT concat('', '') is null; + ?column? +---------- + t +(1 row) + +SELECT concat('Hello', null) is null; + ?column? +---------- + f +(1 row) + +SELECT concat(null, ' World!') is null; + ?column? +---------- + f +(1 row) + +SELECT concat_ws(',', 'ABCDE', 2, NULL, 22); + concat_ws +------------ + ABCDE,2,22 +(1 row) + +SELECT concat_ws('', 'ABCDE', 2, NULL, 22); + concat_ws +----------- + +(1 row) + +SELECT concat_ws(',', '') is null; + ?column? +---------- + t +(1 row) + +SELECT concat_ws('', '') is null; + ?column? +---------- + t +(1 row) + +SELECT left('abcde', 2) is null; + ?column? +---------- + f +(1 row) + +SELECT left('abcde', 0) is null; + ?column? +---------- + t +(1 row) + +SELECT left('abcde', -1) is null; + ?column? +---------- + f +(1 row) + +SELECT left('', 1) is null; + ?column? +---------- + t +(1 row) + +SELECT left('', 0) is null; + ?column? +---------- + t +(1 row) + +SELECT left('', -1) is null; + ?column? +---------- + t +(1 row) + +SELECT right('abcde', 2) is null; + ?column? +---------- + f +(1 row) + +SELECT right('abcde', 0) is null; + ?column? +---------- + t +(1 row) + +SELECT right('abcde', -1) is null; + ?column? +---------- + f +(1 row) + +SELECT right('', 1) is null; + ?column? +---------- + t +(1 row) + +SELECT right('', 0) is null; + ?column? +---------- + t +(1 row) + +SELECT right('', -1) is null; + ?column? +---------- + t +(1 row) + +SELECT format('Hello %s', 'World'); + format +------------- + Hello World +(1 row) + +SELECT format('Hello %s', ''); + format +-------- + Hello +(1 row) + +SELECT format('%s', 'World'); + format +-------- + World +(1 row) + +SELECT format('', 'World'); + format +-------- + +(1 row) + +SELECT format('', ''); + format +-------- + +(1 row) + +SELECT array_to_string(ARRAY[1, 2, 3, NULL, 5], ',') AS RESULT; + result +--------- + 1,2,3,5 +(1 row) + +SELECT array_to_string(ARRAY[1, 2, 3, NULL, 5], '') AS RESULT; + result +-------- + +(1 row) + +SELECT array_to_string(ARRAY[1, 2, 3, NULL, 5], null) AS RESULT; + result +-------- + +(1 row) + +SELECT array_to_string(ARRAY[NULL], '') AS RESULT; + result +-------- + +(1 row) + +SELECT array_to_string(ARRAY[NULL], null) AS RESULT; + result +-------- + +(1 row) + +SELECT array_to_string(ARRAY[1, 2, 3, NULL, 5], ',', '*') AS RESULT; + result +----------- + 1,2,3,*,5 +(1 row) + +SELECT array_to_string(ARRAY[1, 2, 3, NULL, 5], '', '') AS RESULT; + result +-------- + +(1 row) + +SELECT array_to_string(ARRAY[1, 2, 3, NULL, 5], null, null) AS RESULT; + result +-------- + +(1 row) + +SELECT array_to_string(ARRAY[NULL], '', '') AS RESULT; + result +-------- + +(1 row) + +SELECT array_to_string(ARRAY[NULL], null, null) AS RESULT; + result +-------- + +(1 row) + +SELECT nlssort('A', 'nls_sort=schinese_pinyin_m') is null; + ?column? +---------- + f +(1 row) + +SELECT nlssort('', 'nls_sort=schinese_pinyin_m') is null; + ?column? +---------- + t +(1 row) + +SELECT nlssort('A', '') is null; + ?column? +---------- + t +(1 row) + +SELECT convert('text_in_utf8'::bytea, 'UTF8', 'GBK') is null; + ?column? +---------- + f +(1 row) + +SELECT convert(''::bytea, 'UTF8', 'GBK') is null; + ?column? +---------- + t +(1 row) + +SELECT convert('text_in_utf8'::bytea, '', 'GBK') is null; + ?column? +---------- + t +(1 row) + +SELECT convert('text_in_utf8'::bytea, 'UTF8', '') is null; + ?column? +---------- + t +(1 row) + +SELECT convert_from('text_in_utf8'::bytea, 'UTF8') is null; + ?column? +---------- + f +(1 row) + +SELECT convert_from(''::bytea, 'UTF8') is null; + ?column? +---------- + t +(1 row) + +SELECT convert_from('text_in_utf8'::bytea, '') is null; + ?column? +---------- + t +(1 row) + +SELECT convert_to('text_in_utf8'::bytea, 'UTF8') is null; +ERROR: function convert_to(bytea, unknown) does not exist +LINE 1: SELECT convert_to('text_in_utf8'::bytea, 'UTF8') is null; + ^ +HINT: No function matches the given name and argument types. You might need to add explicit type casts. +SELECT convert_to(''::bytea, 'UTF8') is null; +ERROR: function convert_to(bytea, unknown) does not exist +LINE 1: SELECT convert_to(''::bytea, 'UTF8') is null; + ^ +HINT: No function matches the given name and argument types. You might need to add explicit type casts. +SELECT convert_to('text_in_utf8'::bytea, '') is null; +ERROR: function convert_to(bytea, unknown) does not exist +LINE 1: SELECT convert_to('text_in_utf8'::bytea, '') is null; + ^ +HINT: No function matches the given name and argument types. You might need to add explicit type casts. +SELECT md5('ABC') is null; + ?column? +---------- + f +(1 row) + +SELECT md5('') is null; + ?column? +---------- + t +(1 row) + +SELECT sha('ABC') is null; +ERROR: sha/sha1 is supported only in B-format database +SELECT sha('') is null; +ERROR: sha/sha1 is supported only in B-format database +SELECT sha1('ABC') is null; +ERROR: sha/sha1 is supported only in B-format database +SELECT sha1('') is null; +ERROR: sha/sha1 is supported only in B-format database +SELECT sha2('ABC') is null; +ERROR: function sha2(unknown) does not exist +LINE 1: SELECT sha2('ABC') is null; + ^ +HINT: No function matches the given name and argument types. You might need to add explicit type casts. +SELECT sha2('') is null; +ERROR: function sha2(unknown) does not exist +LINE 1: SELECT sha2('') is null; + ^ +HINT: No function matches the given name and argument types. You might need to add explicit type casts. +SELECT decode('MTIzAAE=', 'base64') is null; + ?column? +---------- + f +(1 row) + +SELECT decode('', 'base64') is null; + ?column? +---------- + t +(1 row) + +SELECT decode('MTIzAAE=', '') is null; + ?column? +---------- + t +(1 row) + +select similar_escape('\s+ab','2') is null; + ?column? +---------- + f +(1 row) + +select similar_escape('\s+ab','') is null; + ?column? +---------- + f +(1 row) + +select similar_escape('','2') is null; + ?column? +---------- + t +(1 row) + +select svals('"aa"=>"bb"') is null; + ?column? +---------- + f +(1 row) + +select svals('') is null; + ?column? +---------- +(0 rows) + +select tconvert('aa', 'bb') is null; + ?column? +---------- + f +(1 row) + +select tconvert('', 'bb') is null; + ?column? +---------- + t +(1 row) + +select tconvert('aa', '') is null; + ?column? +---------- + f +(1 row) + +select tconvert('', '') is null; + ?column? +---------- + t +(1 row) + +SELECT encode(E'123\\000\\001', 'base64') is null; + ?column? +---------- + f +(1 row) + +SELECT encode('', 'base64') is null; + ?column? +---------- + t +(1 row) + +SELECT encode(E'123\\000\\001', '') is null; + ?column? +---------- + t +(1 row) + +-- test about vec +CREATE TABLE vec_t1 +( + c varchar +) WITH (ORIENTATION = COLUMN); +CREATE TABLE vec_t2 +( + c varchar not null +) WITH (ORIENTATION = COLUMN); +insert into vec_t1 values(''); +insert into vec_t1 values(' '); +insert into vec_t1 values('abc'); +insert into vec_t2 values(''); +ERROR: null value in column "c" violates not-null constraint +DETAIL: Failing row contains (null). +insert into vec_t2 values(' '); +insert into vec_t2 values('abc'); +select c as input, substr(c, 1, 2) as result, result is null as is_null from vec_t1; + input | result | is_null +-------+--------+--------- + | | t + | | f + abc | ab | f +(3 rows) + +select c as input, substr(c, 1, 1) as result, result is null as is_null from vec_t1; + input | result | is_null +-------+--------+--------- + | | t + | | f + abc | a | f +(3 rows) + +select c as input, substr(c, 0, 2) as result, result is null as is_null from vec_t1; + input | result | is_null +-------+--------+--------- + | | t + | | f + abc | ab | f +(3 rows) + +select c as input, substr(c, 1, 0) as result, result is null as is_null from vec_t1; + input | result | is_null +-------+--------+--------- + | | t + | | t + abc | | t +(3 rows) + +select c as input, substr(c, 1, -1) as result, result is null as is_null from vec_t1; + input | result | is_null +-------+--------+--------- + | | t + | | t + abc | | t +(3 rows) + +delete from vec_t1; +insert into vec_t1 values(''); +insert into vec_t1 values('yzy'); +insert into vec_t1 values('zzz'); +insert into vec_t1 values(' z '); +insert into vec_t1 values(' '); +SELECT c as input, btrim(c, 'y') as result, result is null as isnull from vec_t1; + input | result | isnull +-------+--------+-------- + | | t + yzy | z | f + zzz | zzz | f + z | z | f + | | f +(5 rows) + +SELECT c as input, btrim(c, 'z') as result, result is null as isnull from vec_t1; + input | result | isnull +-------+--------+-------- + | | t + yzy | yzy | f + zzz | | t + z | z | f + | | f +(5 rows) + +SELECT c as input, btrim(c) as result, result is null as isnull from vec_t1; + input | result | isnull +-------+--------+-------- + | | t + yzy | yzy | f + zzz | zzz | f + z | z | f + | | t +(5 rows) + +SELECT c as input, rtrim(c, 'y') as result, result is null as isnull from vec_t1; + input | result | isnull +-------+--------+-------- + | | t + yzy | yz | f + zzz | zzz | f + z | z | f + | | f +(5 rows) + +SELECT c as input, rtrim(c, 'z') as result, result is null as isnull from vec_t1; + input | result | isnull +-------+--------+-------- + | | t + yzy | yzy | f + zzz | | t + z | z | f + | | f +(5 rows) + +SELECT c as input, rtrim(c) as result, result is null as isnull from vec_t1; + input | result | isnull +-------+--------+-------- + | | t + yzy | yzy | f + zzz | zzz | f + z | z | f + | | t +(5 rows) + +-- test row to vec +set try_vector_engine_strategy=force; +create table vec_t3(c text); +insert into vec_t3 values(''); +insert into vec_t3 values(' '); +insert into vec_t3 values('abc'); +explain analyze select c as input, substr(c, 1, 2) as result, result is null as is_null from vec_t3; +--?QUERY PLAN.* +--?.* +--?Row Adapter.* +--?Vector Adapter(type: BATCH MODE).* +--?Seq Scan on vec_t3.* +--?.* +(4 rows) + +select c as input, substr(c, 1, 2) as result, result is null as is_null from vec_t3; + input | result | is_null +-------+--------+--------- + | | t + | | f + abc | ab | f +(3 rows) + +select c as input, substr(c, 1, 1) as result, result is null as is_null from vec_t3; + input | result | is_null +-------+--------+--------- + | | t + | | f + abc | a | f +(3 rows) + +select c as input, substr(c, 0, 2) as result, result is null as is_null from vec_t3; + input | result | is_null +-------+--------+--------- + | | t + | | f + abc | ab | f +(3 rows) + +select c as input, substr(c, 1, 0) as result, result is null as is_null from vec_t3; + input | result | is_null +-------+--------+--------- + | | t + | | t + abc | | t +(3 rows) + +delete from vec_t3; +insert into vec_t3 values(''); +insert into vec_t3 values('yzy'); +insert into vec_t3 values('zzz'); +insert into vec_t3 values(' z '); +insert into vec_t3 values(' '); +explain analyze SELECT c as input, rtrim(c, 'y') as result, result is null as isnull from vec_t3; +--?QUERY PLAN.* +--?.* +--?Row Adapter.* +--?Vector Adapter(type: BATCH MODE).* +--?Seq Scan on vec_t3.* +--?.* +(4 rows) + +SELECT c as input, rtrim(c, 'y') as result, result is null as isnull from vec_t3; + input | result | isnull +-------+--------+-------- + | | t + yzy | yz | f + zzz | zzz | f + z | z | f + | | f +(5 rows) + +SELECT c as input, rtrim(c, 'z') as result, result is null as isnull from vec_t3; + input | result | isnull +-------+--------+-------- + | | t + yzy | yzy | f + zzz | | t + z | z | f + | | f +(5 rows) + +SELECT c as input, rtrim(c) as result, result is null as isnull from vec_t3; + input | result | isnull +-------+--------+-------- + | | t + yzy | yzy | f + zzz | zzz | f + z | z | f + | | t +(5 rows) + +create table pad2_tab(a text, b int); +insert into pad2_tab values('yes', 5), ('yes', 1), ('yes', 0); +create table pad3_tab(a text, b int, c text); +insert into pad3_tab values('yes', 5, 'z'), ('yes', 1, 'z'), ('yes', 0, 'z'); +SELECT a as p1, b as p2, lpad(a, b) is null from pad2_tab; + p1 | p2 | ?column? +-----+----+---------- + yes | 5 | f + yes | 1 | f + yes | 0 | t +(3 rows) + +SELECT a as p1, b as p2, lpad(a, b, c) is null from pad3_tab; + p1 | p2 | ?column? +-----+----+---------- + yes | 5 | f + yes | 1 | f + yes | 0 | t +(3 rows) + +SELECT a as p1, b as p2, rpad(a, b) is null from pad2_tab; + p1 | p2 | ?column? +-----+----+---------- + yes | 5 | f + yes | 1 | f + yes | 0 | t +(3 rows) + +SELECT a as p1, b as p2, rpad(a, b, c) is null from pad3_tab; + p1 | p2 | ?column? +-----+----+---------- + yes | 5 | f + yes | 1 | f + yes | 0 | t +(3 rows) + +create table trim2(a text, b text); +insert into trim2 values('yzy', 'y'), ('zzz', 'z'); +create table trim1(a text); +insert into trim1 values(' z '), (' '); +SELECT a as p1, b as p2, btrim(a, b) is null from trim2; + p1 | p2 | ?column? +-----+----+---------- + yzy | y | f + zzz | z | t +(2 rows) + +SELECT a as p1, b as p2, ltrim(a, b) is null from trim2; + p1 | p2 | ?column? +-----+----+---------- + yzy | y | f + zzz | z | t +(2 rows) + +SELECT a as p1, b as p2, rtrim(a, b) is null from trim2; + p1 | p2 | ?column? +-----+----+---------- + yzy | y | f + zzz | z | t +(2 rows) + +SELECT a as p1, btrim(a) is null from trim1; + p1 | ?column? +-----+---------- + z | f + | t +(2 rows) + +SELECT a as p1, ltrim(a) is null from trim1; + p1 | ?column? +-----+---------- + z | f + | t +(2 rows) + +SELECT a as p1, rtrim(a) is null from trim1; + p1 | ?column? +-----+---------- + z | f + | t +(2 rows) + +create table translate3(a text, b text, c text); +insert into translate3 values('xyx', 'x', 'z'), ('xzx', 'x', ''), ('xxx', 'x', ''), ('xxx', 'x', ' '), ('xxx', '', 'z'), ('', 'x', 'z'); +SELECT a as p1, b as p2, c as p3, translate(a, b, c) is null from translate3; + p1 | p2 | p3 | ?column? +-----+----+----+---------- + xyx | x | z | f + xzx | x | | t + xxx | x | | t + xxx | x | | f + xxx | | z | t + | x | z | t +(6 rows) + +create table repeat2 (a text, b int); +insert into repeat2 values('a', 3), ('a', 0), (' ', 3), ('', 3), ('', 0); +SELECT a as p1, b as p2, repeat(a, b) is null from repeat2; + p1 | p2 | ?column? +----+----+---------- + a | 3 | f + a | 0 | t + | 3 | f + | 3 | t + | 0 | t +(5 rows) + +create table oidvectortypes1 (a oidvector); +insert into oidvectortypes1 values ('123 456'), (''), (' '); +SELECT a as p1, oidvectortypes(a) is null from oidvectortypes1; + p1 | ?column? +---------+---------- + 123 456 | f + | t + | t +(3 rows) + +create table regexp_replace3 (a text, b text, c text); +insert into regexp_replace3 values('Thomas', '.[mN]a.', 'M'), ('omas', '.[mN]a.', ''), ('Thomas', '', 'M'), ('', '.[mN]a.', 'M'), ('omas', '.[mN]a.', ''); +SELECT a as p1, b as p2, c as p3, regexp_replace(a, b, c) is null from regexp_replace3; + p1 | p2 | p3 | ?column? +--------+---------+----+---------- + Thomas | .[mN]a. | M | f + omas | .[mN]a. | | t + Thomas | | M | f + | .[mN]a. | M | t + omas | .[mN]a. | | t +(5 rows) + +create table regexp_replace6 (a text, b text, c text, d int, e int, f text); +insert into regexp_replace6 values ('foobarbaz','b(..)', E'X\\1Y', 2, 2, 'n'), ('bar','b(..)', '', 1, 1, 'n'); +SELECT a as p1, b as p2, c as p3, d as p4, e as p5, f as p6, regexp_replace(a, b, c, d, e, f) is null from regexp_replace6; + p1 | p2 | p3 | p4 | p5 | p6 | ?column? +-----------+-------+------+----+----+----+---------- + foobarbaz | b(..) | X\1Y | 2 | 2 | n | f + bar | b(..) | | 1 | 1 | n | t +(2 rows) + +create table regexp_split_to_table2 (a text, b text); +insert into regexp_split_to_table2 values('hello world', E'\\s+'), ('', E'\\s+'), ('hello world', ''), ('hello world', null); +SELECT a as p1, b as p2, regexp_split_to_table(a, b) is null from regexp_split_to_table2; + p1 | p2 | ?column? +-------------+-----+---------- + hello world | \s+ | f + hello world | \s+ | f +(2 rows) + +create table substr2(a bytea, b int); +insert into substr2 values ('123'::bytea, 3), ('123'::bytea, 4); +select a as p1, b as p2, substr(a, b) is null from substr2; + p1 | p2 | ?column? +----------+----+---------- + \x313233 | 3 | f + \x313233 | 4 | t +(2 rows) + +create table substr3(a bytea, b int, c int); +insert into substr3 values('123'::bytea, 1, 2), ('123'::bytea, 1, 0); +select a as p1, b as p2, c as p3, substr(a, b, c) is null from substr3; + p1 | p2 | p3 | ?column? +----------+----+----+---------- + \x313233 | 1 | 2 | f + \x313233 | 1 | 0 | t +(2 rows) + +create table replace3 (a text, b text, c text); +insert into replace3 values ('abc', 'ab', 'd'), ('abc', 'abc', ''), ('abc', 'abc', ''), ('abc', 'ab', ''), ('abc', 'ab', null), ('abc', '', 'd'), ('abc', null, 'd'), ('', 'ab', 'd'), (null, 'ab', 'd'); +SELECT a as p1, b as p2, c as p3, replace(a, b, c) is null from replace3; + p1 | p2 | p3 | ?column? +-----+-----+----+---------- + abc | ab | d | f + abc | abc | | t + abc | abc | | t + abc | ab | | f + abc | ab | | f + abc | | d | f + abc | | d | f + | ab | d | t + | ab | d | t +(9 rows) + +create table replace2 (a text, b text); +insert into replace2 values('abc', 'ab'), ('abc', 'abc'), ('abc', ''), ('abc', null), ('', 'ab'), (null, 'ab'); +SELECT a as p1, b as p2, replace(a, b) is null from replace2; +ERROR: function returned NULL +create table split_part3 (a text, b text, c int); +insert into split_part3 values('1~2~3', '~', 3), ('1~2~3~', '~', 4), ('1~2~3', '', 1), ('1~2~3', null, 1), ('', '~', 1), (null, '~', 1); +SELECT a as p1, b as p2, c as p3, split_part(a, b, c) is null from split_part3; + p1 | p2 | p3 | ?column? +--------+----+----+---------- + 1~2~3 | ~ | 3 | f + 1~2~3~ | ~ | 4 | t + 1~2~3 | | 1 | t + 1~2~3 | | 1 | t + | ~ | 1 | t + | ~ | 1 | t +(6 rows) + +create table array_to_string2(a integer[], b text); +insert into array_to_string2 values(ARRAY[1, 2, 3, NULL, 5], ','), (ARRAY[1, 2, 3, NULL, 5], ''), (ARRAY[1, 2, 3, NULL, 5], null), (ARRAY[NULL], ''), (ARRAY[NULL], null); +SELECT a as p1, b as p2, array_to_string(a, b) AS RESULT from array_to_string2; + p1 | p2 | result +----------------+----+--------- + {1,2,3,NULL,5} | , | 1,2,3,5 + {1,2,3,NULL,5} | | + {1,2,3,NULL,5} | | + {NULL} | | + {NULL} | | +(5 rows) + +create table array_to_string3(a integer[], b text, c text); +insert into array_to_string3 values(ARRAY[1, 2, 3, NULL, 5], ',', '*'), (ARRAY[1, 2, 3, NULL, 5], '', ''), (ARRAY[1, 2, 3, NULL, 5], null, null), (ARRAY[NULL], '', ''), (ARRAY[NULL], null, null); +SELECT a as p1, b as p2, c as p3, array_to_string(a, b, c) AS RESULT from array_to_string3; + p1 | p2 | p3 | result +----------------+----+----+----------- + {1,2,3,NULL,5} | , | * | 1,2,3,*,5 + {1,2,3,NULL,5} | | | + {1,2,3,NULL,5} | | | + {NULL} | | | + {NULL} | | | +(5 rows) + +set try_vector_engine_strategy=off; diff --git a/src/test/regress/expected/pg_empty_str.out b/src/test/regress/expected/pg_empty_str.out new file mode 100644 index 000000000..4070cb718 --- /dev/null +++ b/src/test/regress/expected/pg_empty_str.out @@ -0,0 +1,1817 @@ +-- test about issue +create database test_pg dbcompatibility 'PG'; +\c test_pg +create schema accept_schema; +set current_schema to 'accept_schema'; +create table tchar(c char(10)); +set behavior_compat_options to 'accept_empty_str'; +insert into tchar values(' '); +select * from tchar where c = ''; + c +------------ + +(1 row) + +select * from tchar where c = ' '; + c +------------ + +(1 row) + +select * from tchar where c is null; + c +--- +(0 rows) + +drop table tchar; +-- test about const str +select '' is null; + ?column? +---------- + f +(1 row) + +select ' ' is null; + ?column? +---------- + f +(1 row) + +select ' abc ' is null; + ?column? +---------- + f +(1 row) + +select length(''); + length +-------- + 0 +(1 row) + +select length(null); + length +-------- + +(1 row) + +select length(' '); + length +-------- + 1 +(1 row) + +select length(' abc '); + length +-------- + 5 +(1 row) + +select '123'::char(3) is null; + ?column? +---------- + f +(1 row) + +select ''::char(3) is null; + ?column? +---------- + f +(1 row) + +select '123'::varchar(3) is null; + ?column? +---------- + f +(1 row) + +select ''::varchar(3) is null; + ?column? +---------- + f +(1 row) + +select '123'::text is null; + ?column? +---------- + f +(1 row) + +select ''::text is null; + ?column? +---------- + f +(1 row) + +select '123'::clob is null; + ?column? +---------- + f +(1 row) + +select ''::clob is null; + ?column? +---------- + f +(1 row) + +select '123'::blob is null; + ?column? +---------- + f +(1 row) + +select ''::blob is null; + ?column? +---------- + f +(1 row) + +select '123'::bytea is null; + ?column? +---------- + f +(1 row) + +select ''::bytea is null; + ?column? +---------- + f +(1 row) + +select '123'::int1 is null; + ?column? +---------- + f +(1 row) + +select ''::int1 is null; +ERROR: invalid input syntax for integer: "" +LINE 1: select ''::int1 is null; + ^ +select '123'::int2 is null; + ?column? +---------- + f +(1 row) + +select ''::int2 is null; +ERROR: invalid input syntax for integer: "" +LINE 1: select ''::int2 is null; + ^ +select '123'::int is null; + ?column? +---------- + f +(1 row) + +select ''::int is null; +ERROR: invalid input syntax for integer: "" +LINE 1: select ''::int is null; + ^ +select '123'::int8 is null; + ?column? +---------- + f +(1 row) + +select ''::int8 is null; +ERROR: invalid input syntax for type bigint: "" +LINE 1: select ''::int8 is null; + ^ +select '123'::float4 is null; + ?column? +---------- + f +(1 row) + +select ''::float4 is null; +ERROR: invalid input syntax for type real: "" +LINE 1: select ''::float4 is null; + ^ +select '123'::float8 is null; + ?column? +---------- + f +(1 row) + +select ''::float8 is null; +ERROR: invalid input syntax for type double precision: "" +LINE 1: select ''::float8 is null; + ^ +select '123'::numeric is null; + ?column? +---------- + f +(1 row) + +select ''::numeric is null; +ERROR: invalid input syntax for type numeric: "" +LINE 1: select ''::numeric is null; + ^ +select ''::date is null; +ERROR: invalid input syntax for type date: "" +LINE 1: select ''::date is null; + ^ +select ''::time is null; +ERROR: invalid input syntax for type time: "" +LINE 1: select ''::time is null; + ^ +select ''::timestamp is null; +ERROR: invalid input syntax for type timestamp: "" +LINE 1: select ''::timestamp is null; + ^ +-- test about var str +create table result_tab ("statement" text, result text); +declare +str_empty text := ''; +str_space text := ' '; +str_num text := '123'; +str text := ' abc '; +begin + insert into result_tab select 'select str_empty is null', str_empty is null; + insert into result_tab select 'select str_space is null', str_space is null; + insert into result_tab select 'select str is null', str is null; + insert into result_tab select 'select length(str_empty)', length(str_empty); + insert into result_tab select 'select length(null)', length(null); + insert into result_tab select 'select length(str_space)', length(str_space); + insert into result_tab select 'select length(str)', length(str); + insert into result_tab select 'select str_num::text is null', str_num::text is null; + insert into result_tab select 'select str_empty::text is null', str_empty::text is null; + insert into result_tab select 'select str_num::bytea is null;', str_num::bytea is null; + insert into result_tab select 'select str_empty::bytea is null', str_empty::bytea is null; + insert into result_tab select 'select str_num::int is null', str_num::int is null; + insert into result_tab select 'select str_num::float8 is null', str_num::float8 is null; + insert into result_tab select 'select str_num::numeric is null', str_num::numeric is null; +end; +/ +select * from result_tab; + statement | result +---------------------------------+-------- + select str_empty is null | false + select str_space is null | false + select str is null | false + select length(str_empty) | 0 + select length(null) | + select length(str_space) | 1 + select length(str) | 5 + select str_num::text is null | false + select str_empty::text is null | false + select str_num::bytea is null; | false + select str_empty::bytea is null | false + select str_num::int is null | false + select str_num::float8 is null | false + select str_num::numeric is null | false +(14 rows) + +-- test about function which return str +SELECT overlay('hello' placing 'world' from 2 for 3 ) is null; + ?column? +---------- + f +(1 row) + +SELECT overlay('hello' placing '' from 1 for 5 ) is null; + ?column? +---------- + f +(1 row) + +SELECT quote_ident('') is null; + ?column? +---------- + f +(1 row) + +SELECT quote_literal('') is null; + ?column? +---------- + f +(1 row) + +SELECT quote_nullable('') is null; + ?column? +---------- + f +(1 row) + +SELECT reverse('') is null; + ?column? +---------- + f +(1 row) + +SELECT ''||'' is null; + ?column? +---------- + false +(1 row) + +SELECT ''||41; + ?column? +---------- + 41 +(1 row) + +SELECT lower('') is null; + ?column? +---------- + f +(1 row) + +SELECT initcap('') is null; + ?column? +---------- + f +(1 row) + +SELECT ascii(''); + ascii +------- + 0 +(1 row) + +SELECT lpad('yes', 5) is null; + ?column? +---------- + f +(1 row) + +SELECT lpad('yes', 1) is null; + ?column? +---------- + f +(1 row) + +SELECT lpad('yes', 0) is null; + ?column? +---------- + f +(1 row) + +SELECT lpad('yes', 5, 'z') is null; + ?column? +---------- + f +(1 row) + +SELECT lpad('yes', 1, 'z') is null; + ?column? +---------- + f +(1 row) + +SELECT lpad('yes', 0, 'z') is null; + ?column? +---------- + f +(1 row) + +SELECT rpad('yes', 5) is null; + ?column? +---------- + f +(1 row) + +SELECT rpad('yes', 1) is null; + ?column? +---------- + f +(1 row) + +SELECT rpad('yes', 0) is null; + ?column? +---------- + f +(1 row) + +SELECT rpad('yes', 5, 'z') is null; + ?column? +---------- + f +(1 row) + +SELECT rpad('yes', 1, 'z') is null; + ?column? +---------- + f +(1 row) + +SELECT rpad('yes', 0, 'z') is null; + ?column? +---------- + f +(1 row) + +SELECT btrim('yzy', 'y') is null; + ?column? +---------- + f +(1 row) + +SELECT btrim('zzz', 'z') is null; + ?column? +---------- + f +(1 row) + +SELECT ltrim('yzy', 'y') is null; + ?column? +---------- + f +(1 row) + +SELECT ltrim('zzz', 'z') is null; + ?column? +---------- + f +(1 row) + +SELECT rtrim('yzy', 'y') is null; + ?column? +---------- + f +(1 row) + +SELECT rtrim('zzz', 'z') is null; + ?column? +---------- + f +(1 row) + +SELECT btrim(' z ') is null; + ?column? +---------- + f +(1 row) + +SELECT btrim(' ') is null; + ?column? +---------- + f +(1 row) + +SELECT ltrim(' z ') is null; + ?column? +---------- + f +(1 row) + +SELECT ltrim(' ') is null; + ?column? +---------- + f +(1 row) + +SELECT rtrim(' z ') is null; + ?column? +---------- + f +(1 row) + +SELECT rtrim(' ') is null; + ?column? +---------- + f +(1 row) + +SELECT translate('xyx', 'x', 'z') is null; + ?column? +---------- + f +(1 row) + +SELECT translate('xzx', 'x', '') is null; + ?column? +---------- + f +(1 row) + +SELECT translate('xxx', 'x', '') is null; + ?column? +---------- + f +(1 row) + +SELECT translate('xxx', 'x', ' ') is null; + ?column? +---------- + f +(1 row) + +SELECT translate('xxx', '', 'z') is null; + ?column? +---------- + f +(1 row) + +SELECT translate('', 'x', 'z') is null; + ?column? +---------- + f +(1 row) + +SELECT repeat('a', 3) is null; + ?column? +---------- + f +(1 row) + +SELECT repeat('a', 0) is null; + ?column? +---------- + f +(1 row) + +SELECT repeat(' ', 3) is null; + ?column? +---------- + f +(1 row) + +SELECT repeat(' ', 0) is null; + ?column? +---------- + f +(1 row) + +SELECT repeat('', 3) is null; + ?column? +---------- + f +(1 row) + +SELECT repeat('', 0) is null; + ?column? +---------- + f +(1 row) + +SELECT pg_catalog.oidvectortypes('123 456') is null; + ?column? +---------- + f +(1 row) + +SELECT pg_catalog.oidvectortypes('') is null; + ?column? +---------- + f +(1 row) + +SELECT pg_catalog.oidvectortypes(' ') is null; + ?column? +---------- + f +(1 row) + +SELECT regexp_replace('Thomas', '.[mN]a.', 'M') is null; + ?column? +---------- + f +(1 row) + +SELECT regexp_replace('omas', '.[mN]a.', ''); + regexp_replace +---------------- + +(1 row) + +SELECT regexp_replace('Thomas', '', 'M') is null; + ?column? +---------- + f +(1 row) + +SELECT regexp_replace('', '.[mN]a.', 'M') is null; + ?column? +---------- + f +(1 row) + +SELECT regexp_replace('omas', '.[mN]a.', '') is null; + ?column? +---------- + f +(1 row) + +SELECT regexp_replace('foobarbaz','b(..)', E'X\\1Y', 2, 2, 'n') is null; + ?column? +---------- + f +(1 row) + +SELECT regexp_replace('bar','b(..)', '', 1, 1, 'n') is null; + ?column? +---------- + f +(1 row) + +SELECT regexp_replace('foobarbaz','b(..)', E'X\\1Y', 'g') is null; + ?column? +---------- + f +(1 row) + +SELECT regexp_replace('abc','abc', '', 'g') is null; + ?column? +---------- + f +(1 row) + +SELECT regexp_substr('str','st') is null; + ?column? +---------- + f +(1 row) + +SELECT regexp_substr('str','[ac]') is null; + ?column? +---------- + t +(1 row) + +SELECT regexp_substr('str','') is null; + ?column? +---------- + f +(1 row) + +SELECT regexp_substr('','st') is null; + ?column? +---------- + t +(1 row) + +SELECT regexp_split_to_table('hello world', E'\\s+') is null; + ?column? +---------- + f + f +(2 rows) + +SELECT regexp_split_to_table('', E'\\s+') is null; + ?column? +---------- + f +(1 row) + +SELECT regexp_split_to_table('hello world', '') is null; + ?column? +---------- + f + f + f + f + f + f + f + f + f + f + f +(11 rows) + +SELECT regexp_split_to_table('hello world', null) is null; + ?column? +---------- +(0 rows) + +SELECT substr('123', 3) is null; + ?column? +---------- + f +(1 row) + +SELECT substr('123', 4) is null; + ?column? +---------- + f +(1 row) + +SELECT substr('123', 1, 2) is null; + ?column? +---------- + f +(1 row) + +SELECT substr('123', 1, 0) is null; + ?column? +---------- + f +(1 row) + +SELECT substr('123', 1, -1) is null; + ?column? +---------- + f +(1 row) + +SELECT substr('123'::bytea, 3) is null; + ?column? +---------- + f +(1 row) + +SELECT substr('123'::bytea, 4) is null; + ?column? +---------- + f +(1 row) + +SELECT substr('123'::bytea, 1, 2) is null; + ?column? +---------- + f +(1 row) + +SELECT substr('123'::bytea, 1, 0) is null; + ?column? +---------- + f +(1 row) + +SELECT substr('123'::bytea, 1, -1) is null; + ?column? +---------- + f +(1 row) + +SELECT substrb('123', 3) is null; + ?column? +---------- + f +(1 row) + +SELECT substrb('123', 4) is null; + ?column? +---------- + f +(1 row) + +SELECT substrb('123', 1, 2) is null; + ?column? +---------- + f +(1 row) + +SELECT substrb('123', 1, 0) is null; + ?column? +---------- + f +(1 row) + +SELECT substrb('123', 1, -1) is null; + ?column? +---------- + f +(1 row) + +SELECT substring('123', 3) is null; + ?column? +---------- + f +(1 row) + +SELECT substring('123', 4) is null; + ?column? +---------- + f +(1 row) + +SELECT substring('123', 1, 2) is null; + ?column? +---------- + f +(1 row) + +SELECT substring('123', 1, 0) is null; + ?column? +---------- + f +(1 row) + +SELECT substring('123', 1, -1) is null; +ERROR: negative substring length not allowed +SELECT substring('123'::bytea, 3) is null; + ?column? +---------- + f +(1 row) + +SELECT substring('123'::bytea, 4) is null; + ?column? +---------- + f +(1 row) + +SELECT substring('123'::bytea, 1, 2) is null; + ?column? +---------- + f +(1 row) + +SELECT substring('123'::bytea, 1, 0) is null; + ?column? +---------- + f +(1 row) + +SELECT substring('123'::bytea, 1, -1) is null; +ERROR: negative substring length not allowed +SELECT replace('abc', 'ab', 'd') is null; + ?column? +---------- + f +(1 row) + +SELECT replace('abc', 'abc', '') is null; + ?column? +---------- + f +(1 row) + +SELECT replace('abc', 'abc', null) is null; + ?column? +---------- + f +(1 row) + +SELECT replace('abc', 'ab', '') is null; + ?column? +---------- + f +(1 row) + +SELECT replace('abc', 'ab', null) is null; + ?column? +---------- + f +(1 row) + +SELECT replace('abc', '', 'd') is null; + ?column? +---------- + f +(1 row) + +SELECT replace('abc', null, 'd') is null; + ?column? +---------- + f +(1 row) + +SELECT replace('', 'ab', 'd') is null; + ?column? +---------- + f +(1 row) + +SELECT replace(null, 'ab', 'd') is null; + ?column? +---------- + t +(1 row) + +SELECT replace('abc', 'ab') is null; + ?column? +---------- + f +(1 row) + +SELECT replace('abc', 'abc') is null; + ?column? +---------- + f +(1 row) + +SELECT replace('abc', '') is null; + ?column? +---------- + f +(1 row) + +SELECT replace('abc', null) is null; + ?column? +---------- + f +(1 row) + +SELECT replace('', 'ab') is null; + ?column? +---------- + f +(1 row) + +SELECT replace(null, 'ab') is null; + ?column? +---------- + t +(1 row) + +SELECT split_part('1~2~3', '~', 3) is null; + ?column? +---------- + f +(1 row) + +SELECT split_part('1~2~3~', '~', 4) is null; + ?column? +---------- + f +(1 row) + +SELECT split_part('1~2~3', '~', -1) is null; +ERROR: field position must be greater than zero +SELECT split_part('1~2~3', '', 1) is null; + ?column? +---------- + f +(1 row) + +SELECT split_part('1~2~3', '', 2) is null; + ?column? +---------- + f +(1 row) + +SELECT split_part('1~2~3', null, 1) is null; + ?column? +---------- + t +(1 row) + +SELECT split_part('', '~', 1) is null; + ?column? +---------- + f +(1 row) + +SELECT split_part(null, '~', 1) is null; + ?column? +---------- + t +(1 row) + +SELECT split_part('1~2~3', '|', 1) is null; + ?column? +---------- + f +(1 row) + +SELECT split_part('1~2~3', '|', 2) is null; + ?column? +---------- + f +(1 row) + +SELECT concat('Hello', ' World!') is null; + ?column? +---------- + f +(1 row) + +SELECT concat('', ' World!') is null; + ?column? +---------- + f +(1 row) + +SELECT concat('Hello', '') is null; + ?column? +---------- + f +(1 row) + +SELECT concat('', '') is null; + ?column? +---------- + f +(1 row) + +SELECT concat('Hello', null) is null; + ?column? +---------- + f +(1 row) + +SELECT concat(null, ' World!') is null; + ?column? +---------- + f +(1 row) + +SELECT concat_ws(',', 'ABCDE', 2, NULL, 22); + concat_ws +------------ + ABCDE,2,22 +(1 row) + +SELECT concat_ws('', 'ABCDE', 2, NULL, 22); + concat_ws +----------- + ABCDE222 +(1 row) + +SELECT concat_ws(',', '') is null; + ?column? +---------- + f +(1 row) + +SELECT concat_ws('', '') is null; + ?column? +---------- + f +(1 row) + +SELECT left('abcde', 2) is null; + ?column? +---------- + f +(1 row) + +SELECT left('abcde', 0) is null; + ?column? +---------- + f +(1 row) + +SELECT left('abcde', -1) is null; + ?column? +---------- + f +(1 row) + +SELECT left('', 1) is null; + ?column? +---------- + f +(1 row) + +SELECT left('', 0) is null; + ?column? +---------- + f +(1 row) + +SELECT left('', -1) is null; + ?column? +---------- + f +(1 row) + +SELECT right('abcde', 2) is null; + ?column? +---------- + f +(1 row) + +SELECT right('abcde', 0) is null; + ?column? +---------- + f +(1 row) + +SELECT right('abcde', -1) is null; + ?column? +---------- + f +(1 row) + +SELECT right('', 1) is null; + ?column? +---------- + f +(1 row) + +SELECT right('', 0) is null; + ?column? +---------- + f +(1 row) + +SELECT right('', -1) is null; + ?column? +---------- + f +(1 row) + +SELECT format('Hello %s', 'World'); + format +------------- + Hello World +(1 row) + +SELECT format('Hello %s', ''); + format +-------- + Hello +(1 row) + +SELECT format('%s', 'World'); + format +-------- + World +(1 row) + +SELECT format('', 'World'); + format +-------- + +(1 row) + +SELECT format('', ''); + format +-------- + +(1 row) + +SELECT array_to_string(ARRAY[1, 2, 3, NULL, 5], ',') AS RESULT; + result +--------- + 1,2,3,5 +(1 row) + +SELECT array_to_string(ARRAY[1, 2, 3, NULL, 5], '') AS RESULT; + result +-------- + 1235 +(1 row) + +SELECT array_to_string(ARRAY[1, 2, 3, NULL, 5], null) AS RESULT; + result +-------- + +(1 row) + +SELECT array_to_string(ARRAY[NULL], '') AS RESULT; + result +-------- + +(1 row) + +SELECT array_to_string(ARRAY[NULL], null) AS RESULT; + result +-------- + +(1 row) + +SELECT array_to_string(ARRAY[1, 2, 3, NULL, 5], ',', '*') AS RESULT; + result +----------- + 1,2,3,*,5 +(1 row) + +SELECT array_to_string(ARRAY[1, 2, 3, NULL, 5], '', '') AS RESULT; + result +-------- + 1235 +(1 row) + +SELECT array_to_string(ARRAY[1, 2, 3, NULL, 5], null, null) AS RESULT; + result +-------- + +(1 row) + +SELECT array_to_string(ARRAY[NULL], '', '') AS RESULT; + result +-------- + +(1 row) + +SELECT array_to_string(ARRAY[NULL], null, null) AS RESULT; + result +-------- + +(1 row) + +SELECT nlssort('A', 'nls_sort=schinese_pinyin_m') is null; + ?column? +---------- + f +(1 row) + +SELECT nlssort('', 'nls_sort=schinese_pinyin_m') is null; + ?column? +---------- + f +(1 row) + +SELECT nlssort('A', '') is null; +ERROR: Sort method is not supported! +DETAIL: Not support the given sort method. +SELECT convert('text_in_utf8'::bytea, 'UTF8', 'GBK') is null; + ?column? +---------- + f +(1 row) + +SELECT convert(''::bytea, 'UTF8', 'GBK') is null; + ?column? +---------- + f +(1 row) + +SELECT convert('text_in_utf8'::bytea, '', 'GBK') is null; +ERROR: invalid source encoding name "" +SELECT convert('text_in_utf8'::bytea, 'UTF8', '') is null; +ERROR: invalid destination encoding name "" +SELECT convert_from('text_in_utf8'::bytea, 'UTF8') is null; + ?column? +---------- + f +(1 row) + +SELECT convert_from(''::bytea, 'UTF8') is null; + ?column? +---------- + f +(1 row) + +SELECT convert_from('text_in_utf8'::bytea, '') is null; +ERROR: invalid source encoding name "" +SELECT convert_to('text_in_utf8'::bytea, 'UTF8') is null; +ERROR: function convert_to(bytea, unknown) does not exist +LINE 1: SELECT convert_to('text_in_utf8'::bytea, 'UTF8') is null; + ^ +HINT: No function matches the given name and argument types. You might need to add explicit type casts. +SELECT convert_to(''::bytea, 'UTF8') is null; +ERROR: function convert_to(bytea, unknown) does not exist +LINE 1: SELECT convert_to(''::bytea, 'UTF8') is null; + ^ +HINT: No function matches the given name and argument types. You might need to add explicit type casts. +SELECT convert_to('text_in_utf8'::bytea, '') is null; +ERROR: function convert_to(bytea, unknown) does not exist +LINE 1: SELECT convert_to('text_in_utf8'::bytea, '') is null; + ^ +HINT: No function matches the given name and argument types. You might need to add explicit type casts. +SELECT md5('ABC') is null; + ?column? +---------- + f +(1 row) + +SELECT md5('') is null; + ?column? +---------- + f +(1 row) + +SELECT sha('ABC') is null; +ERROR: sha/sha1 is supported only in B-format database +SELECT sha('') is null; +ERROR: sha/sha1 is supported only in B-format database +SELECT sha1('ABC') is null; +ERROR: sha/sha1 is supported only in B-format database +SELECT sha1('') is null; +ERROR: sha/sha1 is supported only in B-format database +SELECT sha2('ABC') is null; +ERROR: function sha2(unknown) does not exist +LINE 1: SELECT sha2('ABC') is null; + ^ +HINT: No function matches the given name and argument types. You might need to add explicit type casts. +SELECT sha2('') is null; +ERROR: function sha2(unknown) does not exist +LINE 1: SELECT sha2('') is null; + ^ +HINT: No function matches the given name and argument types. You might need to add explicit type casts. +SELECT decode('MTIzAAE=', 'base64') is null; + ?column? +---------- + f +(1 row) + +SELECT decode('', 'base64') is null; + ?column? +---------- + f +(1 row) + +SELECT decode('MTIzAAE=', '') is null; +ERROR: unrecognized encoding: "" +select similar_escape('\s+ab','2') is null; + ?column? +---------- + f +(1 row) + +select similar_escape('\s+ab','') is null; + ?column? +---------- + f +(1 row) + +select similar_escape('','2') is null; + ?column? +---------- + f +(1 row) + +select svals('"aa"=>"bb"') is null; + ?column? +---------- + f +(1 row) + +select svals('') is null; + ?column? +---------- +(0 rows) + +select tconvert('aa', 'bb') is null; + ?column? +---------- + f +(1 row) + +select tconvert('', 'bb') is null; + ?column? +---------- + f +(1 row) + +select tconvert('aa', '') is null; + ?column? +---------- + f +(1 row) + +select tconvert('', '') is null; + ?column? +---------- + f +(1 row) + +SELECT encode(E'123\\000\\001', 'base64') is null; + ?column? +---------- + f +(1 row) + +SELECT encode('', 'base64') is null; + ?column? +---------- + f +(1 row) + +SELECT encode(E'123\\000\\001', '') is null; +ERROR: unrecognized encoding: "" +-- test about vec +CREATE TABLE vec_t1 +( + c varchar +) WITH (ORIENTATION = COLUMN); +CREATE TABLE vec_t2 +( + c varchar not null +) WITH (ORIENTATION = COLUMN); +insert into vec_t1 values(''); +insert into vec_t1 values(' '); +insert into vec_t1 values('abc'); +insert into vec_t2 values(''); +insert into vec_t2 values(' '); +insert into vec_t2 values('abc'); +select c as input, substr(c, 1, 2) as result, result is null as is_null from vec_t1; + input | result | is_null +-------+--------+--------- + | | f + | | f + abc | ab | f +(3 rows) + +select c as input, substr(c, 1, 1) as result, result is null as is_null from vec_t1; + input | result | is_null +-------+--------+--------- + | | f + | | f + abc | a | f +(3 rows) + +select c as input, substr(c, 0, 2) as result, result is null as is_null from vec_t1; + input | result | is_null +-------+--------+--------- + | | f + | | f + abc | ab | f +(3 rows) + +select c as input, substr(c, 1, 0) as result, result is null as is_null from vec_t1; + input | result | is_null +-------+--------+--------- + | | f + | | f + abc | | f +(3 rows) + +select c as input, substr(c, 1, -1) as result, result is null as is_null from vec_t1; + input | result | is_null +-------+--------+--------- + | | f + | | f + abc | | f +(3 rows) + +delete from vec_t1; +insert into vec_t1 values(''); +insert into vec_t1 values('yzy'); +insert into vec_t1 values('zzz'); +insert into vec_t1 values(' z '); +insert into vec_t1 values(' '); +SELECT c as input, btrim(c, 'y') as result, result is null as isnull from vec_t1; + input | result | isnull +-------+--------+-------- + | | f + yzy | z | f + zzz | zzz | f + z | z | f + | | f +(5 rows) + +SELECT c as input, btrim(c, 'z') as result, result is null as isnull from vec_t1; + input | result | isnull +-------+--------+-------- + | | f + yzy | yzy | f + zzz | | f + z | z | f + | | f +(5 rows) + +SELECT c as input, btrim(c) as result, result is null as isnull from vec_t1; + input | result | isnull +-------+--------+-------- + | | f + yzy | yzy | f + zzz | zzz | f + z | z | f + | | f +(5 rows) + +SELECT c as input, rtrim(c, 'y') as result, result is null as isnull from vec_t1; + input | result | isnull +-------+--------+-------- + | | f + yzy | yz | f + zzz | zzz | f + z | z | f + | | f +(5 rows) + +SELECT c as input, rtrim(c, 'z') as result, result is null as isnull from vec_t1; + input | result | isnull +-------+--------+-------- + | | f + yzy | yzy | f + zzz | | f + z | z | f + | | f +(5 rows) + +SELECT c as input, rtrim(c) as result, result is null as isnull from vec_t1; + input | result | isnull +-------+--------+-------- + | | f + yzy | yzy | f + zzz | zzz | f + z | z | f + | | f +(5 rows) + +-- test row to vec +set try_vector_engine_strategy=force; +create table vec_t3(c text); +insert into vec_t3 values(''); +insert into vec_t3 values(' '); +insert into vec_t3 values('abc'); +explain analyze select c as input, substr(c, 1, 2) as result, result is null as is_null from vec_t3; +--?QUERY PLAN.* +--?.* +--?Row Adapter.* +--?Vector Adapter(type: BATCH MODE).* +--?Seq Scan on vec_t3.* +--?.* +(4 rows) + +select c as input, substr(c, 1, 2) as result, result is null as is_null from vec_t3; + input | result | is_null +-------+--------+--------- + | | f + | | f + abc | ab | f +(3 rows) + +select c as input, substr(c, 1, 1) as result, result is null as is_null from vec_t3; + input | result | is_null +-------+--------+--------- + | | f + | | f + abc | a | f +(3 rows) + +select c as input, substr(c, 0, 2) as result, result is null as is_null from vec_t3; + input | result | is_null +-------+--------+--------- + | | f + | | f + abc | ab | f +(3 rows) + +select c as input, substr(c, 1, 0) as result, result is null as is_null from vec_t3; + input | result | is_null +-------+--------+--------- + | | f + | | f + abc | | f +(3 rows) + +delete from vec_t3; +insert into vec_t3 values(''); +insert into vec_t3 values('yzy'); +insert into vec_t3 values('zzz'); +insert into vec_t3 values(' z '); +insert into vec_t3 values(' '); +explain analyze SELECT c as input, rtrim(c, 'y') as result, result is null as isnull from vec_t3; +--?QUERY PLAN.* +--?.* +--?Row Adapter.* +--?Vector Adapter(type: BATCH MODE).* +--?Seq Scan on vec_t3.* +--?.* +(4 rows) + +SELECT c as input, rtrim(c, 'y') as result, result is null as isnull from vec_t3; + input | result | isnull +-------+--------+-------- + | | f + yzy | yz | f + zzz | zzz | f + z | z | f + | | f +(5 rows) + +SELECT c as input, rtrim(c, 'z') as result, result is null as isnull from vec_t3; + input | result | isnull +-------+--------+-------- + | | f + yzy | yzy | f + zzz | | f + z | z | f + | | f +(5 rows) + +SELECT c as input, rtrim(c) as result, result is null as isnull from vec_t3; + input | result | isnull +-------+--------+-------- + | | f + yzy | yzy | f + zzz | zzz | f + z | z | f + | | f +(5 rows) + +create table pad2_tab(a text, b int); +insert into pad2_tab values('yes', 5), ('yes', 1), ('yes', 0); +create table pad3_tab(a text, b int, c text); +insert into pad3_tab values('yes', 5, 'z'), ('yes', 1, 'z'), ('yes', 0, 'z'); +SELECT a as p1, b as p2, lpad(a, b) is null from pad2_tab; + p1 | p2 | ?column? +-----+----+---------- + yes | 5 | f + yes | 1 | f + yes | 0 | f +(3 rows) + +SELECT a as p1, b as p2, lpad(a, b, c) is null from pad3_tab; + p1 | p2 | ?column? +-----+----+---------- + yes | 5 | f + yes | 1 | f + yes | 0 | f +(3 rows) + +SELECT a as p1, b as p2, rpad(a, b) is null from pad2_tab; + p1 | p2 | ?column? +-----+----+---------- + yes | 5 | f + yes | 1 | f + yes | 0 | f +(3 rows) + +SELECT a as p1, b as p2, rpad(a, b, c) is null from pad3_tab; + p1 | p2 | ?column? +-----+----+---------- + yes | 5 | f + yes | 1 | f + yes | 0 | f +(3 rows) + +create table trim2(a text, b text); +insert into trim2 values('yzy', 'y'), ('zzz', 'z'); +create table trim1(a text); +insert into trim1 values(' z '), (' '); +SELECT a as p1, b as p2, btrim(a, b) is null from trim2; + p1 | p2 | ?column? +-----+----+---------- + yzy | y | f + zzz | z | f +(2 rows) + +SELECT a as p1, b as p2, ltrim(a, b) is null from trim2; + p1 | p2 | ?column? +-----+----+---------- + yzy | y | f + zzz | z | f +(2 rows) + +SELECT a as p1, b as p2, rtrim(a, b) is null from trim2; + p1 | p2 | ?column? +-----+----+---------- + yzy | y | f + zzz | z | f +(2 rows) + +SELECT a as p1, btrim(a) is null from trim1; + p1 | ?column? +-----+---------- + z | f + | f +(2 rows) + +SELECT a as p1, ltrim(a) is null from trim1; + p1 | ?column? +-----+---------- + z | f + | f +(2 rows) + +SELECT a as p1, rtrim(a) is null from trim1; + p1 | ?column? +-----+---------- + z | f + | f +(2 rows) + +create table translate3(a text, b text, c text); +insert into translate3 values('xyx', 'x', 'z'), ('xzx', 'x', ''), ('xxx', 'x', ''), ('xxx', 'x', ' '), ('xxx', '', 'z'), ('', 'x', 'z'); +SELECT a as p1, b as p2, c as p3, translate(a, b, c) is null from translate3; + p1 | p2 | p3 | ?column? +-----+----+----+---------- + xyx | x | z | f + xzx | x | | f + xxx | x | | f + xxx | x | | f + xxx | | z | f + | x | z | f +(6 rows) + +create table repeat2 (a text, b int); +insert into repeat2 values('a', 3), ('a', 0), (' ', 3), ('', 3), ('', 0); +SELECT a as p1, b as p2, repeat(a, b) is null from repeat2; + p1 | p2 | ?column? +----+----+---------- + a | 3 | f + a | 0 | f + | 3 | f + | 3 | f + | 0 | f +(5 rows) + +create table oidvectortypes1 (a oidvector); +insert into oidvectortypes1 values ('123 456'), (''), (' '); +SELECT a as p1, oidvectortypes(a) is null from oidvectortypes1; + p1 | ?column? +---------+---------- + 123 456 | f + | f + | f +(3 rows) + +create table regexp_replace3 (a text, b text, c text); +insert into regexp_replace3 values('Thomas', '.[mN]a.', 'M'), ('omas', '.[mN]a.', ''), ('Thomas', '', 'M'), ('', '.[mN]a.', 'M'), ('omas', '.[mN]a.', ''); +SELECT a as p1, b as p2, c as p3, regexp_replace(a, b, c) is null from regexp_replace3; + p1 | p2 | p3 | ?column? +--------+---------+----+---------- + Thomas | .[mN]a. | M | f + omas | .[mN]a. | | f + Thomas | | M | f + | .[mN]a. | M | f + omas | .[mN]a. | | f +(5 rows) + +create table regexp_replace6 (a text, b text, c text, d int, e int, f text); +insert into regexp_replace6 values ('foobarbaz','b(..)', E'X\\1Y', 2, 2, 'n'), ('bar','b(..)', '', 1, 1, 'n'); +SELECT a as p1, b as p2, c as p3, d as p4, e as p5, f as p6, regexp_replace(a, b, c, d, e, f) is null from regexp_replace6; + p1 | p2 | p3 | p4 | p5 | p6 | ?column? +-----------+-------+------+----+----+----+---------- + foobarbaz | b(..) | X\1Y | 2 | 2 | n | f + bar | b(..) | | 1 | 1 | n | f +(2 rows) + +create table regexp_split_to_table2 (a text, b text); +insert into regexp_split_to_table2 values('hello world', E'\\s+'), ('', E'\\s+'), ('hello world', ''), ('hello world', null); +SELECT a as p1, b as p2, regexp_split_to_table(a, b) is null from regexp_split_to_table2; + p1 | p2 | ?column? +-------------+-----+---------- + hello world | \s+ | f + hello world | \s+ | f + | \s+ | f + hello world | | f + hello world | | f + hello world | | f + hello world | | f + hello world | | f + hello world | | f + hello world | | f + hello world | | f + hello world | | f + hello world | | f + hello world | | f +(14 rows) + +create table substr2(a bytea, b int); +insert into substr2 values ('123'::bytea, 3), ('123'::bytea, 4); +select a as p1, b as p2, substr(a, b) is null from substr2; + p1 | p2 | ?column? +----------+----+---------- + \x313233 | 3 | f + \x313233 | 4 | f +(2 rows) + +create table substr3(a bytea, b int, c int); +insert into substr3 values('123'::bytea, 1, 2), ('123'::bytea, 1, 0); +select a as p1, b as p2, c as p3, substr(a, b, c) is null from substr3; + p1 | p2 | p3 | ?column? +----------+----+----+---------- + \x313233 | 1 | 2 | f + \x313233 | 1 | 0 | f +(2 rows) + +create table replace3 (a text, b text, c text); +insert into replace3 values ('abc', 'ab', 'd'), ('abc', 'abc', ''), ('abc', 'abc', ''), ('abc', 'ab', ''), ('abc', 'ab', null), ('abc', '', 'd'), ('abc', null, 'd'), ('', 'ab', 'd'), (null, 'ab', 'd'); +SELECT a as p1, b as p2, c as p3, replace(a, b, c) is null from replace3; + p1 | p2 | p3 | ?column? +-----+-----+----+---------- + abc | ab | d | f + abc | abc | | f + abc | abc | | f + abc | ab | | f + abc | ab | | f + abc | | d | f + abc | | d | f + | ab | d | f + | ab | d | t +(9 rows) + +create table replace2 (a text, b text); +insert into replace2 values('abc', 'ab'), ('abc', 'abc'), ('abc', ''), ('abc', null), ('', 'ab'), (null, 'ab'); +SELECT a as p1, b as p2, replace(a, b) is null from replace2; + p1 | p2 | ?column? +-----+-----+---------- + abc | ab | f + abc | abc | f + abc | | f + abc | | f + | ab | f + | ab | t +(6 rows) + +create table split_part3 (a text, b text, c int); +insert into split_part3 values('1~2~3', '~', 3), ('1~2~3~', '~', 4), ('1~2~3', '', 1), ('1~2~3', null, 1), ('', '~', 1), (null, '~', 1); +SELECT a as p1, b as p2, c as p3, split_part(a, b, c) is null from split_part3; + p1 | p2 | p3 | ?column? +--------+----+----+---------- + 1~2~3 | ~ | 3 | f + 1~2~3~ | ~ | 4 | f + 1~2~3 | | 1 | f + 1~2~3 | | 1 | t + | ~ | 1 | f + | ~ | 1 | t +(6 rows) + +create table array_to_string2(a integer[], b text); +insert into array_to_string2 values(ARRAY[1, 2, 3, NULL, 5], ','), (ARRAY[1, 2, 3, NULL, 5], ''), (ARRAY[1, 2, 3, NULL, 5], null), (ARRAY[NULL], ''), (ARRAY[NULL], null); +SELECT a as p1, b as p2, array_to_string(a, b) AS RESULT from array_to_string2; + p1 | p2 | result +----------------+----+--------- + {1,2,3,NULL,5} | , | 1,2,3,5 + {1,2,3,NULL,5} | | 1235 + {1,2,3,NULL,5} | | + {NULL} | | + {NULL} | | +(5 rows) + +create table array_to_string3(a integer[], b text, c text); +insert into array_to_string3 values(ARRAY[1, 2, 3, NULL, 5], ',', '*'), (ARRAY[1, 2, 3, NULL, 5], '', ''), (ARRAY[1, 2, 3, NULL, 5], null, null), (ARRAY[NULL], '', ''), (ARRAY[NULL], null, null); +SELECT a as p1, b as p2, c as p3, array_to_string(a, b, c) AS RESULT from array_to_string3; + p1 | p2 | p3 | result +----------------+----+----+----------- + {1,2,3,NULL,5} | , | * | 1,2,3,*,5 + {1,2,3,NULL,5} | | | 1235 + {1,2,3,NULL,5} | | | + {NULL} | | | + {NULL} | | | +(5 rows) + +set try_vector_engine_strategy=off; diff --git a/src/test/regress/input/accept_empty_copy.source b/src/test/regress/input/accept_empty_copy.source new file mode 100644 index 000000000..9761852af --- /dev/null +++ b/src/test/regress/input/accept_empty_copy.source @@ -0,0 +1,33 @@ +create schema accept_schema_copy; +set current_schema to 'accept_schema_copy'; +set behavior_compat_options to 'accept_empty_str'; +create table t ("t_varchar" varchar, "t_text" text, "t_char" char(10), "t_bytea" bytea); +insert into t values('abc', 'abc', 'abc', 'abc'), (' ', ' ', ' ', ' '), ('', '', '', ''); + +create table t_stdin ("t_varchar" varchar, "t_text" text, "t_char" char(10), "t_bytea" bytea); +create table t_csv ("t_varchar" varchar, "t_text" text, "t_char" char(10), "t_bytea" bytea); +create table t_text_tab ("t_varchar" varchar, "t_text" text, "t_char" char(10), "t_bytea" bytea); +create table t_binary ("t_varchar" varchar, "t_text" text, "t_char" char(10), "t_bytea" bytea); + +COPY t TO '@abs_srcdir@/data/datanode1/accept_t_csv.data' WITH(FORMAT 'csv'); +COPY t TO '@abs_srcdir@/data/datanode1/accept_t_text.data' WITH(FORMAT 'text'); +COPY t TO '@abs_srcdir@/data/datanode1/accept_t_binary.data' WITH(FORMAT 'binary'); + +COPY t_csv FROM '@abs_srcdir@/data/datanode1/accept_t_csv.data' WITH(FORMAT 'csv'); +COPY t_text_tab FROM '@abs_srcdir@/data/datanode1/accept_t_text.data' WITH(FORMAT 'text'); +COPY t_binary FROM '@abs_srcdir@/data/datanode1/accept_t_binary.data' WITH(FORMAT 'binary'); + +SELECT * FROM t; +SELECT * FROM t_csv; +SELECT * FROM t_text_tab; +SELECT * FROM t_binary; + +SELECT "t_varchar" is null as "varchar is null", "t_text" is null as "text is null", "t_char" is null as "char is null", "t_bytea" is null as "bytea is null" FROM t; +SELECT "t_varchar" is null as "varchar is null", "t_text" is null as "text is null", "t_char" is null as "char is null", "t_bytea" is null as "bytea is null" FROM t_csv; +SELECT "t_varchar" is null as "varchar is null", "t_text" is null as "text is null", "t_char" is null as "char is null", "t_bytea" is null as "bytea is null" FROM t_text_tab; +SELECT "t_varchar" is null as "varchar is null", "t_text" is null as "text is null", "t_char" is null as "char is null", "t_bytea" is null as "bytea is null" FROM t_binary; + +\d t; +\d t_csv; +\d t_text; +\d t_binary; \ No newline at end of file diff --git a/src/test/regress/input/not_accept_empty_copy.source b/src/test/regress/input/not_accept_empty_copy.source new file mode 100644 index 000000000..9fed5d86e --- /dev/null +++ b/src/test/regress/input/not_accept_empty_copy.source @@ -0,0 +1,33 @@ +create schema not_accept_schema_copy; +set current_schema to 'not_accept_schema_copy'; +set behavior_compat_options to ''; +create table t ("t_varchar" varchar, "t_text" text, "t_char" char(10), "t_bytea" bytea); +insert into t values('abc', 'abc', 'abc', 'abc'), (' ', ' ', ' ', ' '), ('', '', '', ''); + +create table t_stdin ("t_varchar" varchar, "t_text" text, "t_char" char(10), "t_bytea" bytea); +create table t_csv ("t_varchar" varchar, "t_text" text, "t_char" char(10), "t_bytea" bytea); +create table t_text_tab ("t_varchar" varchar, "t_text" text, "t_char" char(10), "t_bytea" bytea); +create table t_binary ("t_varchar" varchar, "t_text" text, "t_char" char(10), "t_bytea" bytea); + +COPY t TO '@abs_srcdir@/data/datanode1/not_accept_t_csv.data' WITH(FORMAT 'csv'); +COPY t TO '@abs_srcdir@/data/datanode1/not_accept_t_text.data' WITH(FORMAT 'text'); +COPY t TO '@abs_srcdir@/data/datanode1/not_accept_t_binary.data' WITH(FORMAT 'binary'); + +COPY t_csv FROM '@abs_srcdir@/data/datanode1/not_accept_t_csv.data' WITH(FORMAT 'csv'); +COPY t_text_tab FROM '@abs_srcdir@/data/datanode1/not_accept_t_text.data' WITH(FORMAT 'text'); +COPY t_binary FROM '@abs_srcdir@/data/datanode1/not_accept_t_binary.data' WITH(FORMAT 'binary'); + +SELECT * FROM t; +SELECT * FROM t_csv; +SELECT * FROM t_text_tab; +SELECT * FROM t_binary; + +SELECT "t_varchar" is null as "varchar is null", "t_text" is null as "text is null", "t_char" is null as "char is null", "t_bytea" is null as "bytea is null" FROM t; +SELECT "t_varchar" is null as "varchar is null", "t_text" is null as "text is null", "t_char" is null as "char is null", "t_bytea" is null as "bytea is null" FROM t_csv; +SELECT "t_varchar" is null as "varchar is null", "t_text" is null as "text is null", "t_char" is null as "char is null", "t_bytea" is null as "bytea is null" FROM t_text_tab; +SELECT "t_varchar" is null as "varchar is null", "t_text" is null as "text is null", "t_char" is null as "char is null", "t_bytea" is null as "bytea is null" FROM t_binary; + +\d t; +\d t_csv; +\d t_text; +\d t_binary; \ No newline at end of file diff --git a/src/test/regress/output/accept_empty_copy.source b/src/test/regress/output/accept_empty_copy.source new file mode 100644 index 000000000..4209534e3 --- /dev/null +++ b/src/test/regress/output/accept_empty_copy.source @@ -0,0 +1,107 @@ +create schema accept_schema_copy; +set current_schema to 'accept_schema_copy'; +set behavior_compat_options to 'accept_empty_str'; +create table t ("t_varchar" varchar, "t_text" text, "t_char" char(10), "t_bytea" bytea); +insert into t values('abc', 'abc', 'abc', 'abc'), (' ', ' ', ' ', ' '), ('', '', '', ''); +create table t_stdin ("t_varchar" varchar, "t_text" text, "t_char" char(10), "t_bytea" bytea); +create table t_csv ("t_varchar" varchar, "t_text" text, "t_char" char(10), "t_bytea" bytea); +create table t_text_tab ("t_varchar" varchar, "t_text" text, "t_char" char(10), "t_bytea" bytea); +create table t_binary ("t_varchar" varchar, "t_text" text, "t_char" char(10), "t_bytea" bytea); +COPY t TO '@abs_srcdir@/data/datanode1/accept_t_csv.data' WITH(FORMAT 'csv'); +COPY t TO '@abs_srcdir@/data/datanode1/accept_t_text.data' WITH(FORMAT 'text'); +COPY t TO '@abs_srcdir@/data/datanode1/accept_t_binary.data' WITH(FORMAT 'binary'); +COPY t_csv FROM '@abs_srcdir@/data/datanode1/accept_t_csv.data' WITH(FORMAT 'csv'); +COPY t_text_tab FROM '@abs_srcdir@/data/datanode1/accept_t_text.data' WITH(FORMAT 'text'); +COPY t_binary FROM '@abs_srcdir@/data/datanode1/accept_t_binary.data' WITH(FORMAT 'binary'); +SELECT * FROM t; + t_varchar | t_text | t_char | t_bytea +-----------+--------+------------+---------- + abc | abc | abc | \x616263 + | | | \x202020 + | | | \x +(3 rows) + +SELECT * FROM t_csv; + t_varchar | t_text | t_char | t_bytea +-----------+--------+------------+---------- + abc | abc | abc | \x616263 + | | | \x202020 + | | | \x +(3 rows) + +SELECT * FROM t_text_tab; + t_varchar | t_text | t_char | t_bytea +-----------+--------+------------+---------- + abc | abc | abc | \x616263 + | | | \x202020 + | | | \x +(3 rows) + +SELECT * FROM t_binary; + t_varchar | t_text | t_char | t_bytea +-----------+--------+------------+---------- + abc | abc | abc | \x616263 + | | | \x202020 + | | | \x +(3 rows) + +SELECT "t_varchar" is null as "varchar is null", "t_text" is null as "text is null", "t_char" is null as "char is null", "t_bytea" is null as "bytea is null" FROM t; + varchar is null | text is null | char is null | bytea is null +-----------------+--------------+--------------+--------------- + f | f | f | f + f | f | f | f + f | f | f | f +(3 rows) + +SELECT "t_varchar" is null as "varchar is null", "t_text" is null as "text is null", "t_char" is null as "char is null", "t_bytea" is null as "bytea is null" FROM t_csv; + varchar is null | text is null | char is null | bytea is null +-----------------+--------------+--------------+--------------- + f | f | f | f + f | f | f | f + f | f | f | f +(3 rows) + +SELECT "t_varchar" is null as "varchar is null", "t_text" is null as "text is null", "t_char" is null as "char is null", "t_bytea" is null as "bytea is null" FROM t_text_tab; + varchar is null | text is null | char is null | bytea is null +-----------------+--------------+--------------+--------------- + f | f | f | f + f | f | f | f + f | f | f | f +(3 rows) + +SELECT "t_varchar" is null as "varchar is null", "t_text" is null as "text is null", "t_char" is null as "char is null", "t_bytea" is null as "bytea is null" FROM t_binary; + varchar is null | text is null | char is null | bytea is null +-----------------+--------------+--------------+--------------- + f | f | f | f + f | f | f | f + f | f | f | f +(3 rows) + +\d t; + Table "accept_schema_copy.t" + Column | Type | Modifiers +-----------+-------------------+----------- + t_varchar | character varying | + t_text | text | + t_char | character(10) | + t_bytea | bytea | + +\d t_csv; + Table "accept_schema_copy.t_csv" + Column | Type | Modifiers +-----------+-------------------+----------- + t_varchar | character varying | + t_text | text | + t_char | character(10) | + t_bytea | bytea | + +\d t_text; +\d t_binary; + Table "accept_schema_copy.t_binary" + Column | Type | Modifiers +-----------+-------------------+----------- + t_varchar | character varying | + t_text | text | + t_char | character(10) | + t_bytea | bytea | + diff --git a/src/test/regress/output/not_accept_empty_copy.source b/src/test/regress/output/not_accept_empty_copy.source new file mode 100644 index 000000000..02288e118 --- /dev/null +++ b/src/test/regress/output/not_accept_empty_copy.source @@ -0,0 +1,107 @@ +create schema not_accept_schema_copy; +set current_schema to 'not_accept_schema_copy'; +set behavior_compat_options to ''; +create table t ("t_varchar" varchar, "t_text" text, "t_char" char(10), "t_bytea" bytea); +insert into t values('abc', 'abc', 'abc', 'abc'), (' ', ' ', ' ', ' '), ('', '', '', ''); +create table t_stdin ("t_varchar" varchar, "t_text" text, "t_char" char(10), "t_bytea" bytea); +create table t_csv ("t_varchar" varchar, "t_text" text, "t_char" char(10), "t_bytea" bytea); +create table t_text_tab ("t_varchar" varchar, "t_text" text, "t_char" char(10), "t_bytea" bytea); +create table t_binary ("t_varchar" varchar, "t_text" text, "t_char" char(10), "t_bytea" bytea); +COPY t TO '@abs_srcdir@/data/datanode1/not_accept_t_csv.data' WITH(FORMAT 'csv'); +COPY t TO '@abs_srcdir@/data/datanode1/not_accept_t_text.data' WITH(FORMAT 'text'); +COPY t TO '@abs_srcdir@/data/datanode1/not_accept_t_binary.data' WITH(FORMAT 'binary'); +COPY t_csv FROM '@abs_srcdir@/data/datanode1/not_accept_t_csv.data' WITH(FORMAT 'csv'); +COPY t_text_tab FROM '@abs_srcdir@/data/datanode1/not_accept_t_text.data' WITH(FORMAT 'text'); +COPY t_binary FROM '@abs_srcdir@/data/datanode1/not_accept_t_binary.data' WITH(FORMAT 'binary'); +SELECT * FROM t; + t_varchar | t_text | t_char | t_bytea +-----------+--------+------------+---------- + abc | abc | abc | \x616263 + | | | \x202020 + | | | +(3 rows) + +SELECT * FROM t_csv; + t_varchar | t_text | t_char | t_bytea +-----------+--------+------------+---------- + abc | abc | abc | \x616263 + | | | \x202020 + | | | +(3 rows) + +SELECT * FROM t_text_tab; + t_varchar | t_text | t_char | t_bytea +-----------+--------+------------+---------- + abc | abc | abc | \x616263 + | | | \x202020 + | | | +(3 rows) + +SELECT * FROM t_binary; + t_varchar | t_text | t_char | t_bytea +-----------+--------+------------+---------- + abc | abc | abc | \x616263 + | | | \x202020 + | | | +(3 rows) + +SELECT "t_varchar" is null as "varchar is null", "t_text" is null as "text is null", "t_char" is null as "char is null", "t_bytea" is null as "bytea is null" FROM t; + varchar is null | text is null | char is null | bytea is null +-----------------+--------------+--------------+--------------- + f | f | f | f + f | f | f | f + t | t | t | t +(3 rows) + +SELECT "t_varchar" is null as "varchar is null", "t_text" is null as "text is null", "t_char" is null as "char is null", "t_bytea" is null as "bytea is null" FROM t_csv; + varchar is null | text is null | char is null | bytea is null +-----------------+--------------+--------------+--------------- + f | f | f | f + f | f | f | f + t | t | t | t +(3 rows) + +SELECT "t_varchar" is null as "varchar is null", "t_text" is null as "text is null", "t_char" is null as "char is null", "t_bytea" is null as "bytea is null" FROM t_text_tab; + varchar is null | text is null | char is null | bytea is null +-----------------+--------------+--------------+--------------- + f | f | f | f + f | f | f | f + t | t | t | t +(3 rows) + +SELECT "t_varchar" is null as "varchar is null", "t_text" is null as "text is null", "t_char" is null as "char is null", "t_bytea" is null as "bytea is null" FROM t_binary; + varchar is null | text is null | char is null | bytea is null +-----------------+--------------+--------------+--------------- + f | f | f | f + f | f | f | f + t | t | t | t +(3 rows) + +\d t; + Table "not_accept_schema_copy.t" + Column | Type | Modifiers +-----------+-------------------+----------- + t_varchar | character varying | + t_text | text | + t_char | character(10) | + t_bytea | bytea | + +\d t_csv; + Table "not_accept_schema_copy.t_csv" + Column | Type | Modifiers +-----------+-------------------+----------- + t_varchar | character varying | + t_text | text | + t_char | character(10) | + t_bytea | bytea | + +\d t_text; +\d t_binary; + Table "not_accept_schema_copy.t_binary" + Column | Type | Modifiers +-----------+-------------------+----------- + t_varchar | character varying | + t_text | text | + t_char | character(10) | + t_bytea | bytea | + diff --git a/src/test/regress/parallel_schedule0A b/src/test/regress/parallel_schedule0A index 55d92ded2..80694e051 100644 --- a/src/test/regress/parallel_schedule0A +++ b/src/test/regress/parallel_schedule0A @@ -456,4 +456,7 @@ test: alter_table_000 alter_table_002 alter_table_003 alter_table_modify test: alter_table_modify_ltt alter_table_modify_gtt test: alter_table_modify alter_table_modify_ustore alter_table_modify_ltt alter_table_modify_gtt +# test for empty string in A format database +test: accept_empty_str not_accept_empty_str pg_empty_str accept_empty_copy not_accept_empty_copy + #test: with \ No newline at end of file diff --git a/src/test/regress/sql/accept_empty_str.sql b/src/test/regress/sql/accept_empty_str.sql new file mode 100644 index 000000000..dd9fb93e4 --- /dev/null +++ b/src/test/regress/sql/accept_empty_str.sql @@ -0,0 +1,405 @@ +-- test about issue +create schema accept_schema; +set current_schema to 'accept_schema'; +create table tchar(c char(10)); +set behavior_compat_options to 'accept_empty_str'; +insert into tchar values(' '); +select * from tchar where c = ''; +select * from tchar where c = ' '; +select * from tchar where c is null; +drop table tchar; + +-- test about const str +select '' is null; +select ' ' is null; +select ' abc ' is null; +select length(''); +select length(null); +select length(' '); +select length(' abc '); +select '123'::char(3) is null; +select ''::char(3) is null; +select '123'::varchar(3) is null; +select ''::varchar(3) is null; +select '123'::text is null; +select ''::text is null; +select '123'::clob is null; +select ''::clob is null; +select '123'::blob is null; +select ''::blob is null; +select '123'::bytea is null; +select ''::bytea is null; +select '123'::int1 is null; +select ''::int1 is null; +select '123'::int2 is null; +select ''::int2 is null; +select '123'::int is null; +select ''::int is null; +select '123'::int8 is null; +select ''::int8 is null; +select '123'::float4 is null; +select ''::float4 is null; +select '123'::float8 is null; +select ''::float8 is null; +select '123'::numeric is null; +select ''::numeric is null; +select ''::date is null; +select ''::time is null; +select ''::timestamp is null; + +-- test about var str +create table result_tab ("statement" text, result text); +declare +str_empty text := ''; +str_space text := ' '; +str_num text := '123'; +str text := ' abc '; +begin + insert into result_tab select 'select str_empty is null', str_empty is null; + insert into result_tab select 'select str_space is null', str_space is null; + insert into result_tab select 'select str is null', str is null; + insert into result_tab select 'select length(str_empty)', length(str_empty); + insert into result_tab select 'select length(null)', length(null); + insert into result_tab select 'select length(str_space)', length(str_space); + insert into result_tab select 'select length(str)', length(str); + insert into result_tab select 'select str_num::text is null', str_num::text is null; + insert into result_tab select 'select str_empty::text is null', str_empty::text is null; + insert into result_tab select 'select str_num::bytea is null;', str_num::bytea is null; + insert into result_tab select 'select str_empty::bytea is null', str_empty::bytea is null; + insert into result_tab select 'select str_num::int is null', str_num::int is null; + insert into result_tab select 'select str_num::float8 is null', str_num::float8 is null; + insert into result_tab select 'select str_num::numeric is null', str_num::numeric is null; +end; +/ + +select * from result_tab; + +-- test about function which return str +SELECT overlay('hello' placing 'world' from 2 for 3 ) is null; +SELECT overlay('hello' placing '' from 1 for 5 ) is null; +SELECT quote_ident('') is null; +SELECT quote_literal('') is null; +SELECT quote_nullable('') is null; +SELECT reverse('') is null; +SELECT ''||'' is null; +SELECT ''||41; +SELECT lower('') is null; +SELECT initcap('') is null; +SELECT ascii(''); +SELECT lpad('yes', 5) is null; +SELECT lpad('yes', 1) is null; +SELECT lpad('yes', 0) is null; +SELECT lpad('yes', 5, 'z') is null; +SELECT lpad('yes', 1, 'z') is null; +SELECT lpad('yes', 0, 'z') is null; +SELECT rpad('yes', 5) is null; +SELECT rpad('yes', 1) is null; +SELECT rpad('yes', 0) is null; +SELECT rpad('yes', 5, 'z') is null; +SELECT rpad('yes', 1, 'z') is null; +SELECT rpad('yes', 0, 'z') is null; +SELECT btrim('yzy', 'y') is null; +SELECT btrim('zzz', 'z') is null; +SELECT ltrim('yzy', 'y') is null; +SELECT ltrim('zzz', 'z') is null; +SELECT rtrim('yzy', 'y') is null; +SELECT rtrim('zzz', 'z') is null; +SELECT btrim(' z ') is null; +SELECT btrim(' ') is null; +SELECT ltrim(' z ') is null; +SELECT ltrim(' ') is null; +SELECT rtrim(' z ') is null; +SELECT rtrim(' ') is null; +SELECT translate('xyx', 'x', 'z') is null; +SELECT translate('xzx', 'x', '') is null; +SELECT translate('xxx', 'x', '') is null; +SELECT translate('xxx', 'x', ' ') is null; +SELECT translate('xxx', '', 'z') is null; +SELECT translate('', 'x', 'z') is null; +SELECT repeat('a', 3) is null; +SELECT repeat('a', 0) is null; +SELECT repeat(' ', 3) is null; +SELECT repeat(' ', 0) is null; +SELECT repeat('', 3) is null; +SELECT repeat('', 0) is null; +SELECT pg_catalog.oidvectortypes('123 456') is null; +SELECT pg_catalog.oidvectortypes('') is null; +SELECT pg_catalog.oidvectortypes(' ') is null; +SELECT regexp_replace('Thomas', '.[mN]a.', 'M') is null; +SELECT regexp_replace('omas', '.[mN]a.', ''); +SELECT regexp_replace('Thomas', '', 'M') is null; +SELECT regexp_replace('', '.[mN]a.', 'M') is null; +SELECT regexp_replace('omas', '.[mN]a.', '') is null; +SELECT regexp_replace('foobarbaz','b(..)', E'X\\1Y', 2, 2, 'n') is null; +SELECT regexp_replace('bar','b(..)', '', 1, 1, 'n') is null; +SELECT regexp_replace('foobarbaz','b(..)', E'X\\1Y', 'g') is null; +SELECT regexp_replace('abc','abc', '', 'g') is null; +SELECT regexp_substr('str','st') is null; +SELECT regexp_substr('str','[ac]') is null; +SELECT regexp_substr('str','') is null; +SELECT regexp_substr('','st') is null; +SELECT regexp_split_to_table('hello world', E'\\s+') is null; +SELECT regexp_split_to_table('', E'\\s+') is null; +SELECT regexp_split_to_table('hello world', '') is null; +SELECT regexp_split_to_table('hello world', null) is null; +SELECT substr('123', 3) is null; +SELECT substr('123', 4) is null; +SELECT substr('123', 1, 2) is null; +SELECT substr('123', 1, 0) is null; +SELECT substr('123', 1, -1) is null; +SELECT substr('123'::bytea, 3) is null; +SELECT substr('123'::bytea, 4) is null; +SELECT substr('123'::bytea, 1, 2) is null; +SELECT substr('123'::bytea, 1, 0) is null; +SELECT substr('123'::bytea, 1, -1) is null; +SELECT substrb('123', 3) is null; +SELECT substrb('123', 4) is null; +SELECT substrb('123', 1, 2) is null; +SELECT substrb('123', 1, 0) is null; +SELECT substrb('123', 1, -1) is null; +SELECT substring('123', 3) is null; +SELECT substring('123', 4) is null; +SELECT substring('123', 1, 2) is null; +SELECT substring('123', 1, 0) is null; +SELECT substring('123', 1, -1) is null; +SELECT substring('123'::bytea, 3) is null; +SELECT substring('123'::bytea, 4) is null; +SELECT substring('123'::bytea, 1, 2) is null; +SELECT substring('123'::bytea, 1, 0) is null; +SELECT substring('123'::bytea, 1, -1) is null; +SELECT replace('abc', 'ab', 'd') is null; +SELECT replace('abc', 'abc', '') is null; +SELECT replace('abc', 'abc', null) is null; +SELECT replace('abc', 'ab', '') is null; +SELECT replace('abc', 'ab', null) is null; +SELECT replace('abc', '', 'd') is null; +SELECT replace('abc', null, 'd') is null; +SELECT replace('', 'ab', 'd') is null; +SELECT replace(null, 'ab', 'd') is null; +SELECT replace('abc', 'ab') is null; +SELECT replace('abc', 'abc') is null; +SELECT replace('abc', '') is null; +SELECT replace('abc', null) is null; +SELECT replace('', 'ab') is null; +SELECT replace(null, 'ab') is null; +SELECT split_part('1~2~3', '~', 3) is null; +SELECT split_part('1~2~3~', '~', 4) is null; +SELECT split_part('1~2~3', '~', -1) is null; +SELECT split_part('1~2~3', '', 1) is null; +SELECT split_part('1~2~3', '', 2) is null; +SELECT split_part('1~2~3', null, 1) is null; +SELECT split_part('', '~', 1) is null; +SELECT split_part(null, '~', 1) is null; +SELECT split_part('1~2~3', '|', 1) is null; +SELECT split_part('1~2~3', '|', 2) is null; +SELECT concat('Hello', ' World!') is null; +SELECT concat('', ' World!') is null; +SELECT concat('Hello', '') is null; +SELECT concat('', '') is null; +SELECT concat('Hello', null) is null; +SELECT concat(null, ' World!') is null; +SELECT concat_ws(',', 'ABCDE', 2, NULL, 22); +SELECT concat_ws('', 'ABCDE', 2, NULL, 22); +SELECT concat_ws(',', '') is null; +SELECT concat_ws('', '') is null; +SELECT left('abcde', 2) is null; +SELECT left('abcde', 0) is null; +SELECT left('abcde', -1) is null; +SELECT left('', 1) is null; +SELECT left('', 0) is null; +SELECT left('', -1) is null; +SELECT right('abcde', 2) is null; +SELECT right('abcde', 0) is null; +SELECT right('abcde', -1) is null; +SELECT right('', 1) is null; +SELECT right('', 0) is null; +SELECT right('', -1) is null; +SELECT format('Hello %s', 'World'); +SELECT format('Hello %s', ''); +SELECT format('%s', 'World'); +SELECT format('', 'World'); +SELECT format('', ''); +SELECT array_to_string(ARRAY[1, 2, 3, NULL, 5], ',') AS RESULT; +SELECT array_to_string(ARRAY[1, 2, 3, NULL, 5], '') AS RESULT; +SELECT array_to_string(ARRAY[1, 2, 3, NULL, 5], null) AS RESULT; +SELECT array_to_string(ARRAY[NULL], '') AS RESULT; +SELECT array_to_string(ARRAY[NULL], null) AS RESULT; +SELECT array_to_string(ARRAY[1, 2, 3, NULL, 5], ',', '*') AS RESULT; +SELECT array_to_string(ARRAY[1, 2, 3, NULL, 5], '', '') AS RESULT; +SELECT array_to_string(ARRAY[1, 2, 3, NULL, 5], null, null) AS RESULT; +SELECT array_to_string(ARRAY[NULL], '', '') AS RESULT; +SELECT array_to_string(ARRAY[NULL], null, null) AS RESULT; +SELECT nlssort('A', 'nls_sort=schinese_pinyin_m') is null; +SELECT nlssort('', 'nls_sort=schinese_pinyin_m') is null; +SELECT nlssort('A', '') is null; +SELECT convert('text_in_utf8'::bytea, 'UTF8', 'GBK') is null; +SELECT convert(''::bytea, 'UTF8', 'GBK') is null; +SELECT convert('text_in_utf8'::bytea, '', 'GBK') is null; +SELECT convert('text_in_utf8'::bytea, 'UTF8', '') is null; +SELECT convert_from('text_in_utf8'::bytea, 'UTF8') is null; +SELECT convert_from(''::bytea, 'UTF8') is null; +SELECT convert_from('text_in_utf8'::bytea, '') is null; +SELECT convert_to('text_in_utf8'::bytea, 'UTF8') is null; +SELECT convert_to(''::bytea, 'UTF8') is null; +SELECT convert_to('text_in_utf8'::bytea, '') is null; +SELECT md5('ABC') is null; +SELECT md5('') is null; +SELECT sha('ABC') is null; +SELECT sha('') is null; +SELECT sha1('ABC') is null; +SELECT sha1('') is null; +SELECT sha2('ABC') is null; +SELECT sha2('') is null; +SELECT decode('MTIzAAE=', 'base64') is null; +SELECT decode('', 'base64') is null; +SELECT decode('MTIzAAE=', '') is null; +select similar_escape('\s+ab','2') is null; +select similar_escape('\s+ab','') is null; +select similar_escape('','2') is null; +select svals('"aa"=>"bb"') is null; +select svals('') is null; +select tconvert('aa', 'bb') is null; +select tconvert('', 'bb') is null; +select tconvert('aa', '') is null; +select tconvert('', '') is null; +SELECT encode(E'123\\000\\001', 'base64') is null; +SELECT encode('', 'base64') is null; +SELECT encode(E'123\\000\\001', '') is null; + +-- test about vec +CREATE TABLE vec_t1 +( + c varchar +) WITH (ORIENTATION = COLUMN); + +CREATE TABLE vec_t2 +( + c varchar not null +) WITH (ORIENTATION = COLUMN); + +insert into vec_t1 values(''); +insert into vec_t1 values(' '); +insert into vec_t1 values('abc'); + +insert into vec_t2 values(''); +insert into vec_t2 values(' '); +insert into vec_t2 values('abc'); + +select c as input, substr(c, 1, 2) as result, result is null as is_null from vec_t1; +select c as input, substr(c, 1, 1) as result, result is null as is_null from vec_t1; +select c as input, substr(c, 0, 2) as result, result is null as is_null from vec_t1; +select c as input, substr(c, 1, 0) as result, result is null as is_null from vec_t1; +select c as input, substr(c, 1, -1) as result, result is null as is_null from vec_t1; + +delete from vec_t1; +insert into vec_t1 values(''); +insert into vec_t1 values('yzy'); +insert into vec_t1 values('zzz'); +insert into vec_t1 values(' z '); +insert into vec_t1 values(' '); + +SELECT c as input, btrim(c, 'y') as result, result is null as isnull from vec_t1; +SELECT c as input, btrim(c, 'z') as result, result is null as isnull from vec_t1; +SELECT c as input, btrim(c) as result, result is null as isnull from vec_t1; +SELECT c as input, rtrim(c, 'y') as result, result is null as isnull from vec_t1; +SELECT c as input, rtrim(c, 'z') as result, result is null as isnull from vec_t1; +SELECT c as input, rtrim(c) as result, result is null as isnull from vec_t1; + +-- test row to vec +set try_vector_engine_strategy=force; +create table vec_t3(c text); +insert into vec_t3 values(''); +insert into vec_t3 values(' '); +insert into vec_t3 values('abc'); + +explain analyze select c as input, substr(c, 1, 2) as result, result is null as is_null from vec_t3; +select c as input, substr(c, 1, 2) as result, result is null as is_null from vec_t3; +select c as input, substr(c, 1, 1) as result, result is null as is_null from vec_t3; +select c as input, substr(c, 0, 2) as result, result is null as is_null from vec_t3; +select c as input, substr(c, 1, 0) as result, result is null as is_null from vec_t3; + +delete from vec_t3; +insert into vec_t3 values(''); +insert into vec_t3 values('yzy'); +insert into vec_t3 values('zzz'); +insert into vec_t3 values(' z '); +insert into vec_t3 values(' '); + +explain analyze SELECT c as input, rtrim(c, 'y') as result, result is null as isnull from vec_t3; +SELECT c as input, rtrim(c, 'y') as result, result is null as isnull from vec_t3; +SELECT c as input, rtrim(c, 'z') as result, result is null as isnull from vec_t3; +SELECT c as input, rtrim(c) as result, result is null as isnull from vec_t3; + +create table pad2_tab(a text, b int); +insert into pad2_tab values('yes', 5), ('yes', 1), ('yes', 0); +create table pad3_tab(a text, b int, c text); +insert into pad3_tab values('yes', 5, 'z'), ('yes', 1, 'z'), ('yes', 0, 'z'); +SELECT a as p1, b as p2, lpad(a, b) is null from pad2_tab; +SELECT a as p1, b as p2, lpad(a, b, c) is null from pad3_tab; +SELECT a as p1, b as p2, rpad(a, b) is null from pad2_tab; +SELECT a as p1, b as p2, rpad(a, b, c) is null from pad3_tab; + +create table trim2(a text, b text); +insert into trim2 values('yzy', 'y'), ('zzz', 'z'); +create table trim1(a text); +insert into trim1 values(' z '), (' '); +SELECT a as p1, b as p2, btrim(a, b) is null from trim2; +SELECT a as p1, b as p2, ltrim(a, b) is null from trim2; +SELECT a as p1, b as p2, rtrim(a, b) is null from trim2; +SELECT a as p1, btrim(a) is null from trim1; +SELECT a as p1, ltrim(a) is null from trim1; +SELECT a as p1, rtrim(a) is null from trim1; + +create table translate3(a text, b text, c text); +insert into translate3 values('xyx', 'x', 'z'), ('xzx', 'x', ''), ('xxx', 'x', ''), ('xxx', 'x', ' '), ('xxx', '', 'z'), ('', 'x', 'z'); +SELECT a as p1, b as p2, c as p3, translate(a, b, c) is null from translate3; + +create table repeat2 (a text, b int); +insert into repeat2 values('a', 3), ('a', 0), (' ', 3), ('', 3), ('', 0); +SELECT a as p1, b as p2, repeat(a, b) is null from repeat2; + +create table oidvectortypes1 (a oidvector); +insert into oidvectortypes1 values ('123 456'), (''), (' '); +SELECT a as p1, oidvectortypes(a) is null from oidvectortypes1; + +create table regexp_replace3 (a text, b text, c text); +insert into regexp_replace3 values('Thomas', '.[mN]a.', 'M'), ('omas', '.[mN]a.', ''), ('Thomas', '', 'M'), ('', '.[mN]a.', 'M'), ('omas', '.[mN]a.', ''); +SELECT a as p1, b as p2, c as p3, regexp_replace(a, b, c) is null from regexp_replace3; + +create table regexp_replace6 (a text, b text, c text, d int, e int, f text); +insert into regexp_replace6 values ('foobarbaz','b(..)', E'X\\1Y', 2, 2, 'n'), ('bar','b(..)', '', 1, 1, 'n'); +SELECT a as p1, b as p2, c as p3, d as p4, e as p5, f as p6, regexp_replace(a, b, c, d, e, f) is null from regexp_replace6; + +create table regexp_split_to_table2 (a text, b text); +insert into regexp_split_to_table2 values('hello world', E'\\s+'), ('', E'\\s+'), ('hello world', ''), ('hello world', null); +SELECT a as p1, b as p2, regexp_split_to_table(a, b) is null from regexp_split_to_table2; + +create table substr2(a bytea, b int); +insert into substr2 values ('123'::bytea, 3), ('123'::bytea, 4); +select a as p1, b as p2, substr(a, b) is null from substr2; + +create table substr3(a bytea, b int, c int); +insert into substr3 values('123'::bytea, 1, 2), ('123'::bytea, 1, 0); +select a as p1, b as p2, c as p3, substr(a, b, c) is null from substr3; + +create table replace3 (a text, b text, c text); +insert into replace3 values ('abc', 'ab', 'd'), ('abc', 'abc', ''), ('abc', 'abc', ''), ('abc', 'ab', ''), ('abc', 'ab', null), ('abc', '', 'd'), ('abc', null, 'd'), ('', 'ab', 'd'), (null, 'ab', 'd'); +SELECT a as p1, b as p2, c as p3, replace(a, b, c) is null from replace3; + +create table replace2 (a text, b text); +insert into replace2 values('abc', 'ab'), ('abc', 'abc'), ('abc', ''), ('abc', null), ('', 'ab'), (null, 'ab'); +SELECT a as p1, b as p2, replace(a, b) is null from replace2; + +create table split_part3 (a text, b text, c int); +insert into split_part3 values('1~2~3', '~', 3), ('1~2~3~', '~', 4), ('1~2~3', '', 1), ('1~2~3', null, 1), ('', '~', 1), (null, '~', 1); +SELECT a as p1, b as p2, c as p3, split_part(a, b, c) is null from split_part3; + +create table array_to_string2(a integer[], b text); +insert into array_to_string2 values(ARRAY[1, 2, 3, NULL, 5], ','), (ARRAY[1, 2, 3, NULL, 5], ''), (ARRAY[1, 2, 3, NULL, 5], null), (ARRAY[NULL], ''), (ARRAY[NULL], null); +SELECT a as p1, b as p2, array_to_string(a, b) AS RESULT from array_to_string2; + +create table array_to_string3(a integer[], b text, c text); +insert into array_to_string3 values(ARRAY[1, 2, 3, NULL, 5], ',', '*'), (ARRAY[1, 2, 3, NULL, 5], '', ''), (ARRAY[1, 2, 3, NULL, 5], null, null), (ARRAY[NULL], '', ''), (ARRAY[NULL], null, null); +SELECT a as p1, b as p2, c as p3, array_to_string(a, b, c) AS RESULT from array_to_string3; + +set try_vector_engine_strategy=off; \ No newline at end of file diff --git a/src/test/regress/sql/not_accept_empty_str.sql b/src/test/regress/sql/not_accept_empty_str.sql new file mode 100644 index 000000000..becb9b5fd --- /dev/null +++ b/src/test/regress/sql/not_accept_empty_str.sql @@ -0,0 +1,405 @@ +-- test about issue +create schema not_accept_schema; +set current_schema to 'not_accept_schema'; +create table tchar(c char(10)); +set behavior_compat_options to ''; +insert into tchar values(' '); +select * from tchar where c = ''; +select * from tchar where c = ' '; +select * from tchar where c is null; +drop table tchar; + +-- test about const str +select '' is null; +select ' ' is null; +select ' abc ' is null; +select length(''); +select length(null); +select length(' '); +select length(' abc '); +select '123'::char(3) is null; +select ''::char(3) is null; +select '123'::varchar(3) is null; +select ''::varchar(3) is null; +select '123'::text is null; +select ''::text is null; +select '123'::clob is null; +select ''::clob is null; +select '123'::blob is null; +select ''::blob is null; +select '123'::bytea is null; +select ''::bytea is null; +select '123'::int1 is null; +select ''::int1 is null; +select '123'::int2 is null; +select ''::int2 is null; +select '123'::int is null; +select ''::int is null; +select '123'::int8 is null; +select ''::int8 is null; +select '123'::float4 is null; +select ''::float4 is null; +select '123'::float8 is null; +select ''::float8 is null; +select '123'::numeric is null; +select ''::numeric is null; +select ''::date is null; +select ''::time is null; +select ''::timestamp is null; + +-- test about var str +create table result_tab ("statement" text, result text); +declare +str_empty text := ''; +str_space text := ' '; +str_num text := '123'; +str text := ' abc '; +begin + insert into result_tab select 'select str_empty is null', str_empty is null; + insert into result_tab select 'select str_space is null', str_space is null; + insert into result_tab select 'select str is null', str is null; + insert into result_tab select 'select length(str_empty)', length(str_empty); + insert into result_tab select 'select length(null)', length(null); + insert into result_tab select 'select length(str_space)', length(str_space); + insert into result_tab select 'select length(str)', length(str); + insert into result_tab select 'select str_num::text is null', str_num::text is null; + insert into result_tab select 'select str_empty::text is null', str_empty::text is null; + insert into result_tab select 'select str_num::bytea is null;', str_num::bytea is null; + insert into result_tab select 'select str_empty::bytea is null', str_empty::bytea is null; + insert into result_tab select 'select str_num::int is null', str_num::int is null; + insert into result_tab select 'select str_num::float8 is null', str_num::float8 is null; + insert into result_tab select 'select str_num::numeric is null', str_num::numeric is null; +end; +/ + +select * from result_tab; + +-- test about function which return str +SELECT overlay('hello' placing 'world' from 2 for 3 ) is null; +SELECT overlay('hello' placing '' from 1 for 5 ) is null; +SELECT quote_ident('') is null; +SELECT quote_literal('') is null; +SELECT quote_nullable('') is null; +SELECT reverse('') is null; +SELECT ''||'' is null; +SELECT ''||41; +SELECT lower('') is null; +SELECT initcap('') is null; +SELECT ascii(''); +SELECT lpad('yes', 5) is null; +SELECT lpad('yes', 1) is null; +SELECT lpad('yes', 0) is null; +SELECT lpad('yes', 5, 'z') is null; +SELECT lpad('yes', 1, 'z') is null; +SELECT lpad('yes', 0, 'z') is null; +SELECT rpad('yes', 5) is null; +SELECT rpad('yes', 1) is null; +SELECT rpad('yes', 0) is null; +SELECT rpad('yes', 5, 'z') is null; +SELECT rpad('yes', 1, 'z') is null; +SELECT rpad('yes', 0, 'z') is null; +SELECT btrim('yzy', 'y') is null; +SELECT btrim('zzz', 'z') is null; +SELECT ltrim('yzy', 'y') is null; +SELECT ltrim('zzz', 'z') is null; +SELECT rtrim('yzy', 'y') is null; +SELECT rtrim('zzz', 'z') is null; +SELECT btrim(' z ') is null; +SELECT btrim(' ') is null; +SELECT ltrim(' z ') is null; +SELECT ltrim(' ') is null; +SELECT rtrim(' z ') is null; +SELECT rtrim(' ') is null; +SELECT translate('xyx', 'x', 'z') is null; +SELECT translate('xzx', 'x', '') is null; +SELECT translate('xxx', 'x', '') is null; +SELECT translate('xxx', 'x', ' ') is null; +SELECT translate('xxx', '', 'z') is null; +SELECT translate('', 'x', 'z') is null; +SELECT repeat('a', 3) is null; +SELECT repeat('a', 0) is null; +SELECT repeat(' ', 3) is null; +SELECT repeat(' ', 0) is null; +SELECT repeat('', 3) is null; +SELECT repeat('', 0) is null; +SELECT pg_catalog.oidvectortypes('123 456') is null; +SELECT pg_catalog.oidvectortypes('') is null; +SELECT pg_catalog.oidvectortypes(' ') is null; +SELECT regexp_replace('Thomas', '.[mN]a.', 'M') is null; +SELECT regexp_replace('omas', '.[mN]a.', ''); +SELECT regexp_replace('Thomas', '', 'M') is null; +SELECT regexp_replace('', '.[mN]a.', 'M') is null; +SELECT regexp_replace('omas', '.[mN]a.', '') is null; +SELECT regexp_replace('foobarbaz','b(..)', E'X\\1Y', 2, 2, 'n') is null; +SELECT regexp_replace('bar','b(..)', '', 1, 1, 'n') is null; +SELECT regexp_replace('foobarbaz','b(..)', E'X\\1Y', 'g') is null; +SELECT regexp_replace('abc','abc', '', 'g') is null; +SELECT regexp_substr('str','st') is null; +SELECT regexp_substr('str','[ac]') is null; +SELECT regexp_substr('str','') is null; +SELECT regexp_substr('','st') is null; +SELECT regexp_split_to_table('hello world', E'\\s+') is null; +SELECT regexp_split_to_table('', E'\\s+') is null; +SELECT regexp_split_to_table('hello world', '') is null; +SELECT regexp_split_to_table('hello world', null) is null; +SELECT substr('123', 3) is null; +SELECT substr('123', 4) is null; +SELECT substr('123', 1, 2) is null; +SELECT substr('123', 1, 0) is null; +SELECT substr('123', 1, -1) is null; +SELECT substr('123'::bytea, 3) is null; +SELECT substr('123'::bytea, 4) is null; +SELECT substr('123'::bytea, 1, 2) is null; +SELECT substr('123'::bytea, 1, 0) is null; +SELECT substr('123'::bytea, 1, -1) is null; +SELECT substrb('123', 3) is null; +SELECT substrb('123', 4) is null; +SELECT substrb('123', 1, 2) is null; +SELECT substrb('123', 1, 0) is null; +SELECT substrb('123', 1, -1) is null; +SELECT substring('123', 3) is null; +SELECT substring('123', 4) is null; +SELECT substring('123', 1, 2) is null; +SELECT substring('123', 1, 0) is null; +SELECT substring('123', 1, -1) is null; +SELECT substring('123'::bytea, 3) is null; +SELECT substring('123'::bytea, 4) is null; +SELECT substring('123'::bytea, 1, 2) is null; +SELECT substring('123'::bytea, 1, 0) is null; +SELECT substring('123'::bytea, 1, -1) is null; +SELECT replace('abc', 'ab', 'd') is null; +SELECT replace('abc', 'abc', '') is null; +SELECT replace('abc', 'abc', null) is null; +SELECT replace('abc', 'ab', '') is null; +SELECT replace('abc', 'ab', null) is null; +SELECT replace('abc', '', 'd') is null; +SELECT replace('abc', null, 'd') is null; +SELECT replace('', 'ab', 'd') is null; +SELECT replace(null, 'ab', 'd') is null; +SELECT replace('abc', 'ab') is null; +SELECT replace('abc', 'abc') is null; +SELECT replace('abc', '') is null; +SELECT replace('abc', null) is null; +SELECT replace('', 'ab') is null; +SELECT replace(null, 'ab') is null; +SELECT split_part('1~2~3', '~', 3) is null; +SELECT split_part('1~2~3~', '~', 4) is null; +SELECT split_part('1~2~3', '~', -1) is null; +SELECT split_part('1~2~3', '', 1) is null; +SELECT split_part('1~2~3', '', 2) is null; +SELECT split_part('1~2~3', null, 1) is null; +SELECT split_part('', '~', 1) is null; +SELECT split_part(null, '~', 1) is null; +SELECT split_part('1~2~3', '|', 1) is null; +SELECT split_part('1~2~3', '|', 2) is null; +SELECT concat('Hello', ' World!') is null; +SELECT concat('', ' World!') is null; +SELECT concat('Hello', '') is null; +SELECT concat('', '') is null; +SELECT concat('Hello', null) is null; +SELECT concat(null, ' World!') is null; +SELECT concat_ws(',', 'ABCDE', 2, NULL, 22); +SELECT concat_ws('', 'ABCDE', 2, NULL, 22); +SELECT concat_ws(',', '') is null; +SELECT concat_ws('', '') is null; +SELECT left('abcde', 2) is null; +SELECT left('abcde', 0) is null; +SELECT left('abcde', -1) is null; +SELECT left('', 1) is null; +SELECT left('', 0) is null; +SELECT left('', -1) is null; +SELECT right('abcde', 2) is null; +SELECT right('abcde', 0) is null; +SELECT right('abcde', -1) is null; +SELECT right('', 1) is null; +SELECT right('', 0) is null; +SELECT right('', -1) is null; +SELECT format('Hello %s', 'World'); +SELECT format('Hello %s', ''); +SELECT format('%s', 'World'); +SELECT format('', 'World'); +SELECT format('', ''); +SELECT array_to_string(ARRAY[1, 2, 3, NULL, 5], ',') AS RESULT; +SELECT array_to_string(ARRAY[1, 2, 3, NULL, 5], '') AS RESULT; +SELECT array_to_string(ARRAY[1, 2, 3, NULL, 5], null) AS RESULT; +SELECT array_to_string(ARRAY[NULL], '') AS RESULT; +SELECT array_to_string(ARRAY[NULL], null) AS RESULT; +SELECT array_to_string(ARRAY[1, 2, 3, NULL, 5], ',', '*') AS RESULT; +SELECT array_to_string(ARRAY[1, 2, 3, NULL, 5], '', '') AS RESULT; +SELECT array_to_string(ARRAY[1, 2, 3, NULL, 5], null, null) AS RESULT; +SELECT array_to_string(ARRAY[NULL], '', '') AS RESULT; +SELECT array_to_string(ARRAY[NULL], null, null) AS RESULT; +SELECT nlssort('A', 'nls_sort=schinese_pinyin_m') is null; +SELECT nlssort('', 'nls_sort=schinese_pinyin_m') is null; +SELECT nlssort('A', '') is null; +SELECT convert('text_in_utf8'::bytea, 'UTF8', 'GBK') is null; +SELECT convert(''::bytea, 'UTF8', 'GBK') is null; +SELECT convert('text_in_utf8'::bytea, '', 'GBK') is null; +SELECT convert('text_in_utf8'::bytea, 'UTF8', '') is null; +SELECT convert_from('text_in_utf8'::bytea, 'UTF8') is null; +SELECT convert_from(''::bytea, 'UTF8') is null; +SELECT convert_from('text_in_utf8'::bytea, '') is null; +SELECT convert_to('text_in_utf8'::bytea, 'UTF8') is null; +SELECT convert_to(''::bytea, 'UTF8') is null; +SELECT convert_to('text_in_utf8'::bytea, '') is null; +SELECT md5('ABC') is null; +SELECT md5('') is null; +SELECT sha('ABC') is null; +SELECT sha('') is null; +SELECT sha1('ABC') is null; +SELECT sha1('') is null; +SELECT sha2('ABC') is null; +SELECT sha2('') is null; +SELECT decode('MTIzAAE=', 'base64') is null; +SELECT decode('', 'base64') is null; +SELECT decode('MTIzAAE=', '') is null; +select similar_escape('\s+ab','2') is null; +select similar_escape('\s+ab','') is null; +select similar_escape('','2') is null; +select svals('"aa"=>"bb"') is null; +select svals('') is null; +select tconvert('aa', 'bb') is null; +select tconvert('', 'bb') is null; +select tconvert('aa', '') is null; +select tconvert('', '') is null; +SELECT encode(E'123\\000\\001', 'base64') is null; +SELECT encode('', 'base64') is null; +SELECT encode(E'123\\000\\001', '') is null; + +-- test about vec +CREATE TABLE vec_t1 +( + c varchar +) WITH (ORIENTATION = COLUMN); + +CREATE TABLE vec_t2 +( + c varchar not null +) WITH (ORIENTATION = COLUMN); + +insert into vec_t1 values(''); +insert into vec_t1 values(' '); +insert into vec_t1 values('abc'); + +insert into vec_t2 values(''); +insert into vec_t2 values(' '); +insert into vec_t2 values('abc'); + +select c as input, substr(c, 1, 2) as result, result is null as is_null from vec_t1; +select c as input, substr(c, 1, 1) as result, result is null as is_null from vec_t1; +select c as input, substr(c, 0, 2) as result, result is null as is_null from vec_t1; +select c as input, substr(c, 1, 0) as result, result is null as is_null from vec_t1; +select c as input, substr(c, 1, -1) as result, result is null as is_null from vec_t1; + +delete from vec_t1; +insert into vec_t1 values(''); +insert into vec_t1 values('yzy'); +insert into vec_t1 values('zzz'); +insert into vec_t1 values(' z '); +insert into vec_t1 values(' '); + +SELECT c as input, btrim(c, 'y') as result, result is null as isnull from vec_t1; +SELECT c as input, btrim(c, 'z') as result, result is null as isnull from vec_t1; +SELECT c as input, btrim(c) as result, result is null as isnull from vec_t1; +SELECT c as input, rtrim(c, 'y') as result, result is null as isnull from vec_t1; +SELECT c as input, rtrim(c, 'z') as result, result is null as isnull from vec_t1; +SELECT c as input, rtrim(c) as result, result is null as isnull from vec_t1; + +-- test row to vec +set try_vector_engine_strategy=force; +create table vec_t3(c text); +insert into vec_t3 values(''); +insert into vec_t3 values(' '); +insert into vec_t3 values('abc'); + +explain analyze select c as input, substr(c, 1, 2) as result, result is null as is_null from vec_t3; +select c as input, substr(c, 1, 2) as result, result is null as is_null from vec_t3; +select c as input, substr(c, 1, 1) as result, result is null as is_null from vec_t3; +select c as input, substr(c, 0, 2) as result, result is null as is_null from vec_t3; +select c as input, substr(c, 1, 0) as result, result is null as is_null from vec_t3; + +delete from vec_t3; +insert into vec_t3 values(''); +insert into vec_t3 values('yzy'); +insert into vec_t3 values('zzz'); +insert into vec_t3 values(' z '); +insert into vec_t3 values(' '); + +explain analyze SELECT c as input, rtrim(c, 'y') as result, result is null as isnull from vec_t3; +SELECT c as input, rtrim(c, 'y') as result, result is null as isnull from vec_t3; +SELECT c as input, rtrim(c, 'z') as result, result is null as isnull from vec_t3; +SELECT c as input, rtrim(c) as result, result is null as isnull from vec_t3; + +create table pad2_tab(a text, b int); +insert into pad2_tab values('yes', 5), ('yes', 1), ('yes', 0); +create table pad3_tab(a text, b int, c text); +insert into pad3_tab values('yes', 5, 'z'), ('yes', 1, 'z'), ('yes', 0, 'z'); +SELECT a as p1, b as p2, lpad(a, b) is null from pad2_tab; +SELECT a as p1, b as p2, lpad(a, b, c) is null from pad3_tab; +SELECT a as p1, b as p2, rpad(a, b) is null from pad2_tab; +SELECT a as p1, b as p2, rpad(a, b, c) is null from pad3_tab; + +create table trim2(a text, b text); +insert into trim2 values('yzy', 'y'), ('zzz', 'z'); +create table trim1(a text); +insert into trim1 values(' z '), (' '); +SELECT a as p1, b as p2, btrim(a, b) is null from trim2; +SELECT a as p1, b as p2, ltrim(a, b) is null from trim2; +SELECT a as p1, b as p2, rtrim(a, b) is null from trim2; +SELECT a as p1, btrim(a) is null from trim1; +SELECT a as p1, ltrim(a) is null from trim1; +SELECT a as p1, rtrim(a) is null from trim1; + +create table translate3(a text, b text, c text); +insert into translate3 values('xyx', 'x', 'z'), ('xzx', 'x', ''), ('xxx', 'x', ''), ('xxx', 'x', ' '), ('xxx', '', 'z'), ('', 'x', 'z'); +SELECT a as p1, b as p2, c as p3, translate(a, b, c) is null from translate3; + +create table repeat2 (a text, b int); +insert into repeat2 values('a', 3), ('a', 0), (' ', 3), ('', 3), ('', 0); +SELECT a as p1, b as p2, repeat(a, b) is null from repeat2; + +create table oidvectortypes1 (a oidvector); +insert into oidvectortypes1 values ('123 456'), (''), (' '); +SELECT a as p1, oidvectortypes(a) is null from oidvectortypes1; + +create table regexp_replace3 (a text, b text, c text); +insert into regexp_replace3 values('Thomas', '.[mN]a.', 'M'), ('omas', '.[mN]a.', ''), ('Thomas', '', 'M'), ('', '.[mN]a.', 'M'), ('omas', '.[mN]a.', ''); +SELECT a as p1, b as p2, c as p3, regexp_replace(a, b, c) is null from regexp_replace3; + +create table regexp_replace6 (a text, b text, c text, d int, e int, f text); +insert into regexp_replace6 values ('foobarbaz','b(..)', E'X\\1Y', 2, 2, 'n'), ('bar','b(..)', '', 1, 1, 'n'); +SELECT a as p1, b as p2, c as p3, d as p4, e as p5, f as p6, regexp_replace(a, b, c, d, e, f) is null from regexp_replace6; + +create table regexp_split_to_table2 (a text, b text); +insert into regexp_split_to_table2 values('hello world', E'\\s+'), ('', E'\\s+'), ('hello world', ''), ('hello world', null); +SELECT a as p1, b as p2, regexp_split_to_table(a, b) is null from regexp_split_to_table2; + +create table substr2(a bytea, b int); +insert into substr2 values ('123'::bytea, 3), ('123'::bytea, 4); +select a as p1, b as p2, substr(a, b) is null from substr2; + +create table substr3(a bytea, b int, c int); +insert into substr3 values('123'::bytea, 1, 2), ('123'::bytea, 1, 0); +select a as p1, b as p2, c as p3, substr(a, b, c) is null from substr3; + +create table replace3 (a text, b text, c text); +insert into replace3 values ('abc', 'ab', 'd'), ('abc', 'abc', ''), ('abc', 'abc', ''), ('abc', 'ab', ''), ('abc', 'ab', null), ('abc', '', 'd'), ('abc', null, 'd'), ('', 'ab', 'd'), (null, 'ab', 'd'); +SELECT a as p1, b as p2, c as p3, replace(a, b, c) is null from replace3; + +create table replace2 (a text, b text); +insert into replace2 values('abc', 'ab'), ('abc', 'abc'), ('abc', ''), ('abc', null), ('', 'ab'), (null, 'ab'); +SELECT a as p1, b as p2, replace(a, b) is null from replace2; + +create table split_part3 (a text, b text, c int); +insert into split_part3 values('1~2~3', '~', 3), ('1~2~3~', '~', 4), ('1~2~3', '', 1), ('1~2~3', null, 1), ('', '~', 1), (null, '~', 1); +SELECT a as p1, b as p2, c as p3, split_part(a, b, c) is null from split_part3; + +create table array_to_string2(a integer[], b text); +insert into array_to_string2 values(ARRAY[1, 2, 3, NULL, 5], ','), (ARRAY[1, 2, 3, NULL, 5], ''), (ARRAY[1, 2, 3, NULL, 5], null), (ARRAY[NULL], ''), (ARRAY[NULL], null); +SELECT a as p1, b as p2, array_to_string(a, b) AS RESULT from array_to_string2; + +create table array_to_string3(a integer[], b text, c text); +insert into array_to_string3 values(ARRAY[1, 2, 3, NULL, 5], ',', '*'), (ARRAY[1, 2, 3, NULL, 5], '', ''), (ARRAY[1, 2, 3, NULL, 5], null, null), (ARRAY[NULL], '', ''), (ARRAY[NULL], null, null); +SELECT a as p1, b as p2, c as p3, array_to_string(a, b, c) AS RESULT from array_to_string3; + +set try_vector_engine_strategy=off; \ No newline at end of file diff --git a/src/test/regress/sql/pg_empty_str.sql b/src/test/regress/sql/pg_empty_str.sql new file mode 100644 index 000000000..24f4bdea5 --- /dev/null +++ b/src/test/regress/sql/pg_empty_str.sql @@ -0,0 +1,407 @@ +-- test about issue +create database test_pg dbcompatibility 'PG'; +\c test_pg +create schema accept_schema; +set current_schema to 'accept_schema'; +create table tchar(c char(10)); +set behavior_compat_options to 'accept_empty_str'; +insert into tchar values(' '); +select * from tchar where c = ''; +select * from tchar where c = ' '; +select * from tchar where c is null; +drop table tchar; + +-- test about const str +select '' is null; +select ' ' is null; +select ' abc ' is null; +select length(''); +select length(null); +select length(' '); +select length(' abc '); +select '123'::char(3) is null; +select ''::char(3) is null; +select '123'::varchar(3) is null; +select ''::varchar(3) is null; +select '123'::text is null; +select ''::text is null; +select '123'::clob is null; +select ''::clob is null; +select '123'::blob is null; +select ''::blob is null; +select '123'::bytea is null; +select ''::bytea is null; +select '123'::int1 is null; +select ''::int1 is null; +select '123'::int2 is null; +select ''::int2 is null; +select '123'::int is null; +select ''::int is null; +select '123'::int8 is null; +select ''::int8 is null; +select '123'::float4 is null; +select ''::float4 is null; +select '123'::float8 is null; +select ''::float8 is null; +select '123'::numeric is null; +select ''::numeric is null; +select ''::date is null; +select ''::time is null; +select ''::timestamp is null; + +-- test about var str +create table result_tab ("statement" text, result text); +declare +str_empty text := ''; +str_space text := ' '; +str_num text := '123'; +str text := ' abc '; +begin + insert into result_tab select 'select str_empty is null', str_empty is null; + insert into result_tab select 'select str_space is null', str_space is null; + insert into result_tab select 'select str is null', str is null; + insert into result_tab select 'select length(str_empty)', length(str_empty); + insert into result_tab select 'select length(null)', length(null); + insert into result_tab select 'select length(str_space)', length(str_space); + insert into result_tab select 'select length(str)', length(str); + insert into result_tab select 'select str_num::text is null', str_num::text is null; + insert into result_tab select 'select str_empty::text is null', str_empty::text is null; + insert into result_tab select 'select str_num::bytea is null;', str_num::bytea is null; + insert into result_tab select 'select str_empty::bytea is null', str_empty::bytea is null; + insert into result_tab select 'select str_num::int is null', str_num::int is null; + insert into result_tab select 'select str_num::float8 is null', str_num::float8 is null; + insert into result_tab select 'select str_num::numeric is null', str_num::numeric is null; +end; +/ + +select * from result_tab; + +-- test about function which return str +SELECT overlay('hello' placing 'world' from 2 for 3 ) is null; +SELECT overlay('hello' placing '' from 1 for 5 ) is null; +SELECT quote_ident('') is null; +SELECT quote_literal('') is null; +SELECT quote_nullable('') is null; +SELECT reverse('') is null; +SELECT ''||'' is null; +SELECT ''||41; +SELECT lower('') is null; +SELECT initcap('') is null; +SELECT ascii(''); +SELECT lpad('yes', 5) is null; +SELECT lpad('yes', 1) is null; +SELECT lpad('yes', 0) is null; +SELECT lpad('yes', 5, 'z') is null; +SELECT lpad('yes', 1, 'z') is null; +SELECT lpad('yes', 0, 'z') is null; +SELECT rpad('yes', 5) is null; +SELECT rpad('yes', 1) is null; +SELECT rpad('yes', 0) is null; +SELECT rpad('yes', 5, 'z') is null; +SELECT rpad('yes', 1, 'z') is null; +SELECT rpad('yes', 0, 'z') is null; +SELECT btrim('yzy', 'y') is null; +SELECT btrim('zzz', 'z') is null; +SELECT ltrim('yzy', 'y') is null; +SELECT ltrim('zzz', 'z') is null; +SELECT rtrim('yzy', 'y') is null; +SELECT rtrim('zzz', 'z') is null; +SELECT btrim(' z ') is null; +SELECT btrim(' ') is null; +SELECT ltrim(' z ') is null; +SELECT ltrim(' ') is null; +SELECT rtrim(' z ') is null; +SELECT rtrim(' ') is null; +SELECT translate('xyx', 'x', 'z') is null; +SELECT translate('xzx', 'x', '') is null; +SELECT translate('xxx', 'x', '') is null; +SELECT translate('xxx', 'x', ' ') is null; +SELECT translate('xxx', '', 'z') is null; +SELECT translate('', 'x', 'z') is null; +SELECT repeat('a', 3) is null; +SELECT repeat('a', 0) is null; +SELECT repeat(' ', 3) is null; +SELECT repeat(' ', 0) is null; +SELECT repeat('', 3) is null; +SELECT repeat('', 0) is null; +SELECT pg_catalog.oidvectortypes('123 456') is null; +SELECT pg_catalog.oidvectortypes('') is null; +SELECT pg_catalog.oidvectortypes(' ') is null; +SELECT regexp_replace('Thomas', '.[mN]a.', 'M') is null; +SELECT regexp_replace('omas', '.[mN]a.', ''); +SELECT regexp_replace('Thomas', '', 'M') is null; +SELECT regexp_replace('', '.[mN]a.', 'M') is null; +SELECT regexp_replace('omas', '.[mN]a.', '') is null; +SELECT regexp_replace('foobarbaz','b(..)', E'X\\1Y', 2, 2, 'n') is null; +SELECT regexp_replace('bar','b(..)', '', 1, 1, 'n') is null; +SELECT regexp_replace('foobarbaz','b(..)', E'X\\1Y', 'g') is null; +SELECT regexp_replace('abc','abc', '', 'g') is null; +SELECT regexp_substr('str','st') is null; +SELECT regexp_substr('str','[ac]') is null; +SELECT regexp_substr('str','') is null; +SELECT regexp_substr('','st') is null; +SELECT regexp_split_to_table('hello world', E'\\s+') is null; +SELECT regexp_split_to_table('', E'\\s+') is null; +SELECT regexp_split_to_table('hello world', '') is null; +SELECT regexp_split_to_table('hello world', null) is null; +SELECT substr('123', 3) is null; +SELECT substr('123', 4) is null; +SELECT substr('123', 1, 2) is null; +SELECT substr('123', 1, 0) is null; +SELECT substr('123', 1, -1) is null; +SELECT substr('123'::bytea, 3) is null; +SELECT substr('123'::bytea, 4) is null; +SELECT substr('123'::bytea, 1, 2) is null; +SELECT substr('123'::bytea, 1, 0) is null; +SELECT substr('123'::bytea, 1, -1) is null; +SELECT substrb('123', 3) is null; +SELECT substrb('123', 4) is null; +SELECT substrb('123', 1, 2) is null; +SELECT substrb('123', 1, 0) is null; +SELECT substrb('123', 1, -1) is null; +SELECT substring('123', 3) is null; +SELECT substring('123', 4) is null; +SELECT substring('123', 1, 2) is null; +SELECT substring('123', 1, 0) is null; +SELECT substring('123', 1, -1) is null; +SELECT substring('123'::bytea, 3) is null; +SELECT substring('123'::bytea, 4) is null; +SELECT substring('123'::bytea, 1, 2) is null; +SELECT substring('123'::bytea, 1, 0) is null; +SELECT substring('123'::bytea, 1, -1) is null; +SELECT replace('abc', 'ab', 'd') is null; +SELECT replace('abc', 'abc', '') is null; +SELECT replace('abc', 'abc', null) is null; +SELECT replace('abc', 'ab', '') is null; +SELECT replace('abc', 'ab', null) is null; +SELECT replace('abc', '', 'd') is null; +SELECT replace('abc', null, 'd') is null; +SELECT replace('', 'ab', 'd') is null; +SELECT replace(null, 'ab', 'd') is null; +SELECT replace('abc', 'ab') is null; +SELECT replace('abc', 'abc') is null; +SELECT replace('abc', '') is null; +SELECT replace('abc', null) is null; +SELECT replace('', 'ab') is null; +SELECT replace(null, 'ab') is null; +SELECT split_part('1~2~3', '~', 3) is null; +SELECT split_part('1~2~3~', '~', 4) is null; +SELECT split_part('1~2~3', '~', -1) is null; +SELECT split_part('1~2~3', '', 1) is null; +SELECT split_part('1~2~3', '', 2) is null; +SELECT split_part('1~2~3', null, 1) is null; +SELECT split_part('', '~', 1) is null; +SELECT split_part(null, '~', 1) is null; +SELECT split_part('1~2~3', '|', 1) is null; +SELECT split_part('1~2~3', '|', 2) is null; +SELECT concat('Hello', ' World!') is null; +SELECT concat('', ' World!') is null; +SELECT concat('Hello', '') is null; +SELECT concat('', '') is null; +SELECT concat('Hello', null) is null; +SELECT concat(null, ' World!') is null; +SELECT concat_ws(',', 'ABCDE', 2, NULL, 22); +SELECT concat_ws('', 'ABCDE', 2, NULL, 22); +SELECT concat_ws(',', '') is null; +SELECT concat_ws('', '') is null; +SELECT left('abcde', 2) is null; +SELECT left('abcde', 0) is null; +SELECT left('abcde', -1) is null; +SELECT left('', 1) is null; +SELECT left('', 0) is null; +SELECT left('', -1) is null; +SELECT right('abcde', 2) is null; +SELECT right('abcde', 0) is null; +SELECT right('abcde', -1) is null; +SELECT right('', 1) is null; +SELECT right('', 0) is null; +SELECT right('', -1) is null; +SELECT format('Hello %s', 'World'); +SELECT format('Hello %s', ''); +SELECT format('%s', 'World'); +SELECT format('', 'World'); +SELECT format('', ''); +SELECT array_to_string(ARRAY[1, 2, 3, NULL, 5], ',') AS RESULT; +SELECT array_to_string(ARRAY[1, 2, 3, NULL, 5], '') AS RESULT; +SELECT array_to_string(ARRAY[1, 2, 3, NULL, 5], null) AS RESULT; +SELECT array_to_string(ARRAY[NULL], '') AS RESULT; +SELECT array_to_string(ARRAY[NULL], null) AS RESULT; +SELECT array_to_string(ARRAY[1, 2, 3, NULL, 5], ',', '*') AS RESULT; +SELECT array_to_string(ARRAY[1, 2, 3, NULL, 5], '', '') AS RESULT; +SELECT array_to_string(ARRAY[1, 2, 3, NULL, 5], null, null) AS RESULT; +SELECT array_to_string(ARRAY[NULL], '', '') AS RESULT; +SELECT array_to_string(ARRAY[NULL], null, null) AS RESULT; +SELECT nlssort('A', 'nls_sort=schinese_pinyin_m') is null; +SELECT nlssort('', 'nls_sort=schinese_pinyin_m') is null; +SELECT nlssort('A', '') is null; +SELECT convert('text_in_utf8'::bytea, 'UTF8', 'GBK') is null; +SELECT convert(''::bytea, 'UTF8', 'GBK') is null; +SELECT convert('text_in_utf8'::bytea, '', 'GBK') is null; +SELECT convert('text_in_utf8'::bytea, 'UTF8', '') is null; +SELECT convert_from('text_in_utf8'::bytea, 'UTF8') is null; +SELECT convert_from(''::bytea, 'UTF8') is null; +SELECT convert_from('text_in_utf8'::bytea, '') is null; +SELECT convert_to('text_in_utf8'::bytea, 'UTF8') is null; +SELECT convert_to(''::bytea, 'UTF8') is null; +SELECT convert_to('text_in_utf8'::bytea, '') is null; +SELECT md5('ABC') is null; +SELECT md5('') is null; +SELECT sha('ABC') is null; +SELECT sha('') is null; +SELECT sha1('ABC') is null; +SELECT sha1('') is null; +SELECT sha2('ABC') is null; +SELECT sha2('') is null; +SELECT decode('MTIzAAE=', 'base64') is null; +SELECT decode('', 'base64') is null; +SELECT decode('MTIzAAE=', '') is null; +select similar_escape('\s+ab','2') is null; +select similar_escape('\s+ab','') is null; +select similar_escape('','2') is null; +select svals('"aa"=>"bb"') is null; +select svals('') is null; +select tconvert('aa', 'bb') is null; +select tconvert('', 'bb') is null; +select tconvert('aa', '') is null; +select tconvert('', '') is null; +SELECT encode(E'123\\000\\001', 'base64') is null; +SELECT encode('', 'base64') is null; +SELECT encode(E'123\\000\\001', '') is null; + +-- test about vec +CREATE TABLE vec_t1 +( + c varchar +) WITH (ORIENTATION = COLUMN); + +CREATE TABLE vec_t2 +( + c varchar not null +) WITH (ORIENTATION = COLUMN); + +insert into vec_t1 values(''); +insert into vec_t1 values(' '); +insert into vec_t1 values('abc'); + +insert into vec_t2 values(''); +insert into vec_t2 values(' '); +insert into vec_t2 values('abc'); + +select c as input, substr(c, 1, 2) as result, result is null as is_null from vec_t1; +select c as input, substr(c, 1, 1) as result, result is null as is_null from vec_t1; +select c as input, substr(c, 0, 2) as result, result is null as is_null from vec_t1; +select c as input, substr(c, 1, 0) as result, result is null as is_null from vec_t1; +select c as input, substr(c, 1, -1) as result, result is null as is_null from vec_t1; + +delete from vec_t1; +insert into vec_t1 values(''); +insert into vec_t1 values('yzy'); +insert into vec_t1 values('zzz'); +insert into vec_t1 values(' z '); +insert into vec_t1 values(' '); + +SELECT c as input, btrim(c, 'y') as result, result is null as isnull from vec_t1; +SELECT c as input, btrim(c, 'z') as result, result is null as isnull from vec_t1; +SELECT c as input, btrim(c) as result, result is null as isnull from vec_t1; +SELECT c as input, rtrim(c, 'y') as result, result is null as isnull from vec_t1; +SELECT c as input, rtrim(c, 'z') as result, result is null as isnull from vec_t1; +SELECT c as input, rtrim(c) as result, result is null as isnull from vec_t1; + +-- test row to vec +set try_vector_engine_strategy=force; +create table vec_t3(c text); +insert into vec_t3 values(''); +insert into vec_t3 values(' '); +insert into vec_t3 values('abc'); + +explain analyze select c as input, substr(c, 1, 2) as result, result is null as is_null from vec_t3; +select c as input, substr(c, 1, 2) as result, result is null as is_null from vec_t3; +select c as input, substr(c, 1, 1) as result, result is null as is_null from vec_t3; +select c as input, substr(c, 0, 2) as result, result is null as is_null from vec_t3; +select c as input, substr(c, 1, 0) as result, result is null as is_null from vec_t3; + +delete from vec_t3; +insert into vec_t3 values(''); +insert into vec_t3 values('yzy'); +insert into vec_t3 values('zzz'); +insert into vec_t3 values(' z '); +insert into vec_t3 values(' '); + +explain analyze SELECT c as input, rtrim(c, 'y') as result, result is null as isnull from vec_t3; +SELECT c as input, rtrim(c, 'y') as result, result is null as isnull from vec_t3; +SELECT c as input, rtrim(c, 'z') as result, result is null as isnull from vec_t3; +SELECT c as input, rtrim(c) as result, result is null as isnull from vec_t3; + +create table pad2_tab(a text, b int); +insert into pad2_tab values('yes', 5), ('yes', 1), ('yes', 0); +create table pad3_tab(a text, b int, c text); +insert into pad3_tab values('yes', 5, 'z'), ('yes', 1, 'z'), ('yes', 0, 'z'); +SELECT a as p1, b as p2, lpad(a, b) is null from pad2_tab; +SELECT a as p1, b as p2, lpad(a, b, c) is null from pad3_tab; +SELECT a as p1, b as p2, rpad(a, b) is null from pad2_tab; +SELECT a as p1, b as p2, rpad(a, b, c) is null from pad3_tab; + +create table trim2(a text, b text); +insert into trim2 values('yzy', 'y'), ('zzz', 'z'); +create table trim1(a text); +insert into trim1 values(' z '), (' '); +SELECT a as p1, b as p2, btrim(a, b) is null from trim2; +SELECT a as p1, b as p2, ltrim(a, b) is null from trim2; +SELECT a as p1, b as p2, rtrim(a, b) is null from trim2; +SELECT a as p1, btrim(a) is null from trim1; +SELECT a as p1, ltrim(a) is null from trim1; +SELECT a as p1, rtrim(a) is null from trim1; + +create table translate3(a text, b text, c text); +insert into translate3 values('xyx', 'x', 'z'), ('xzx', 'x', ''), ('xxx', 'x', ''), ('xxx', 'x', ' '), ('xxx', '', 'z'), ('', 'x', 'z'); +SELECT a as p1, b as p2, c as p3, translate(a, b, c) is null from translate3; + +create table repeat2 (a text, b int); +insert into repeat2 values('a', 3), ('a', 0), (' ', 3), ('', 3), ('', 0); +SELECT a as p1, b as p2, repeat(a, b) is null from repeat2; + +create table oidvectortypes1 (a oidvector); +insert into oidvectortypes1 values ('123 456'), (''), (' '); +SELECT a as p1, oidvectortypes(a) is null from oidvectortypes1; + +create table regexp_replace3 (a text, b text, c text); +insert into regexp_replace3 values('Thomas', '.[mN]a.', 'M'), ('omas', '.[mN]a.', ''), ('Thomas', '', 'M'), ('', '.[mN]a.', 'M'), ('omas', '.[mN]a.', ''); +SELECT a as p1, b as p2, c as p3, regexp_replace(a, b, c) is null from regexp_replace3; + +create table regexp_replace6 (a text, b text, c text, d int, e int, f text); +insert into regexp_replace6 values ('foobarbaz','b(..)', E'X\\1Y', 2, 2, 'n'), ('bar','b(..)', '', 1, 1, 'n'); +SELECT a as p1, b as p2, c as p3, d as p4, e as p5, f as p6, regexp_replace(a, b, c, d, e, f) is null from regexp_replace6; + +create table regexp_split_to_table2 (a text, b text); +insert into regexp_split_to_table2 values('hello world', E'\\s+'), ('', E'\\s+'), ('hello world', ''), ('hello world', null); +SELECT a as p1, b as p2, regexp_split_to_table(a, b) is null from regexp_split_to_table2; + +create table substr2(a bytea, b int); +insert into substr2 values ('123'::bytea, 3), ('123'::bytea, 4); +select a as p1, b as p2, substr(a, b) is null from substr2; + +create table substr3(a bytea, b int, c int); +insert into substr3 values('123'::bytea, 1, 2), ('123'::bytea, 1, 0); +select a as p1, b as p2, c as p3, substr(a, b, c) is null from substr3; + +create table replace3 (a text, b text, c text); +insert into replace3 values ('abc', 'ab', 'd'), ('abc', 'abc', ''), ('abc', 'abc', ''), ('abc', 'ab', ''), ('abc', 'ab', null), ('abc', '', 'd'), ('abc', null, 'd'), ('', 'ab', 'd'), (null, 'ab', 'd'); +SELECT a as p1, b as p2, c as p3, replace(a, b, c) is null from replace3; + +create table replace2 (a text, b text); +insert into replace2 values('abc', 'ab'), ('abc', 'abc'), ('abc', ''), ('abc', null), ('', 'ab'), (null, 'ab'); +SELECT a as p1, b as p2, replace(a, b) is null from replace2; + +create table split_part3 (a text, b text, c int); +insert into split_part3 values('1~2~3', '~', 3), ('1~2~3~', '~', 4), ('1~2~3', '', 1), ('1~2~3', null, 1), ('', '~', 1), (null, '~', 1); +SELECT a as p1, b as p2, c as p3, split_part(a, b, c) is null from split_part3; + +create table array_to_string2(a integer[], b text); +insert into array_to_string2 values(ARRAY[1, 2, 3, NULL, 5], ','), (ARRAY[1, 2, 3, NULL, 5], ''), (ARRAY[1, 2, 3, NULL, 5], null), (ARRAY[NULL], ''), (ARRAY[NULL], null); +SELECT a as p1, b as p2, array_to_string(a, b) AS RESULT from array_to_string2; + +create table array_to_string3(a integer[], b text, c text); +insert into array_to_string3 values(ARRAY[1, 2, 3, NULL, 5], ',', '*'), (ARRAY[1, 2, 3, NULL, 5], '', ''), (ARRAY[1, 2, 3, NULL, 5], null, null), (ARRAY[NULL], '', ''), (ARRAY[NULL], null, null); +SELECT a as p1, b as p2, c as p3, array_to_string(a, b, c) AS RESULT from array_to_string3; + +set try_vector_engine_strategy=off; \ No newline at end of file From 38c4b34c03d3da1b40ee73b44d0fbc2fcc8b0f0f Mon Sep 17 00:00:00 2001 From: chenxiaobin19 <1025221611@qq.com> Date: Thu, 20 Jul 2023 17:18:12 +0800 Subject: [PATCH 038/304] =?UTF-8?q?5.1.0=E9=80=82=E9=85=8Ddolphin=E6=8F=92?= =?UTF-8?q?=E4=BB=B6=E5=8D=87=E7=BA=A7=E5=88=B02.0.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CMakeLists.txt | 12 ++++-- build/script/aarch64_lite_list | 8 ++-- build/script/aarch64_opengauss_list | 8 ++-- .../opengauss_release_list_ubuntu_single | 8 ++-- build/script/x86_64_lite_list | 8 ++-- build/script/x86_64_opengauss_list | 8 ++-- .../rollback-post_catalog_maindb_92_899.sql | 41 ++++++++++++++++++- .../rollback-post_catalog_otherdb_92_899.sql | 41 ++++++++++++++++++- .../upgrade-post_catalog_maindb_92_899.sql | 4 +- .../upgrade-post_catalog_otherdb_92_899.sql | 4 +- 10 files changed, 116 insertions(+), 26 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ce0ba9dd3..caa9c1f43 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,16 +111,22 @@ if(EXISTS ${CMAKE_SOURCE_DIR}/contrib/dolphin) install(FILES ${CMAKE_SOURCE_DIR}/contrib/dolphin/dolphin.control DESTINATION share/postgresql/extension/ ) - install(FILES ${CMAKE_SOURCE_DIR}/contrib/dolphin/dolphin--1.1.sql + install(FILES ${CMAKE_SOURCE_DIR}/contrib/dolphin/dolphin--2.0.sql DESTINATION share/postgresql/extension/ ) install(FILES ${CMAKE_SOURCE_DIR}/contrib/dolphin/dolphin--1.0.sql DESTINATION share/postgresql/extension/ ) - install(FILES ${CMAKE_SOURCE_DIR}/contrib/dolphin/upgrade_script/dolphin--1.0--1.1.sql + install(FILES ${CMAKE_SOURCE_DIR}/contrib/dolphin/dolphin--1.0--2.0.sql DESTINATION share/postgresql/extension/ ) - install(FILES ${CMAKE_SOURCE_DIR}/contrib/dolphin/rollback_script/dolphin--1.1--1.0.sql + install(FILES ${CMAKE_SOURCE_DIR}/contrib/dolphin/dolphin--2.0--1.0.sql + DESTINATION share/postgresql/extension/ + ) + install(FILES ${CMAKE_SOURCE_DIR}/contrib/dolphin/dolphin--1.1--2.0.sql + DESTINATION share/postgresql/extension/ + ) + install(FILES ${CMAKE_SOURCE_DIR}/contrib/dolphin/dolphin--2.0--1.1.sql DESTINATION share/postgresql/extension/ ) install(FILES ${CMAKE_SOURCE_DIR}/contrib/dolphin/openGauss_expr_dolphin.ir diff --git a/build/script/aarch64_lite_list b/build/script/aarch64_lite_list index 714a2fe9f..3d6e75783 100644 --- a/build/script/aarch64_lite_list +++ b/build/script/aarch64_lite_list @@ -33,10 +33,12 @@ ./share/postgresql/extension/security_plugin.control ./share/postgresql/extension/security_plugin--1.0.sql ./share/postgresql/extension/dolphin.control -./share/postgresql/extension/dolphin--1.1.sql +./share/postgresql/extension/dolphin--2.0.sql ./share/postgresql/extension/dolphin--1.0.sql -./share/postgresql/extension/dolphin--1.0--1.1.sql -./share/postgresql/extension/dolphin--1.1--1.0.sql +./share/postgresql/extension/dolphin--1.0--2.0.sql +./share/postgresql/extension/dolphin--2.0--1.0.sql +./share/postgresql/extension/dolphin--1.1--2.0.sql +./share/postgresql/extension/dolphin--2.0--1.1.sql ./share/postgresql/extension/openGauss_expr_dolphin.ir ./share/postgresql/extension/file_fdw--1.0.sql ./share/postgresql/extension/plpgsql.control diff --git a/build/script/aarch64_opengauss_list b/build/script/aarch64_opengauss_list index fd36832e0..811419bb9 100644 --- a/build/script/aarch64_opengauss_list +++ b/build/script/aarch64_opengauss_list @@ -69,10 +69,12 @@ ./share/postgresql/extension/ndpplugin.control ./share/postgresql/extension/ndpplugin--1.0.sql ./share/postgresql/extension/dolphin.control +./share/postgresql/extension/dolphin--2.0.sql ./share/postgresql/extension/dolphin--1.0.sql -./share/postgresql/extension/dolphin--1.1.sql -./share/postgresql/extension/dolphin--1.0--1.1.sql -./share/postgresql/extension/dolphin--1.1--1.0.sql +./share/postgresql/extension/dolphin--1.0--2.0.sql +./share/postgresql/extension/dolphin--2.0--1.0.sql +./share/postgresql/extension/dolphin--1.1--2.0.sql +./share/postgresql/extension/dolphin--2.0--1.1.sql ./share/postgresql/extension/openGauss_expr_dolphin.ir ./share/postgresql/extension/file_fdw--1.0.sql ./share/postgresql/extension/plpgsql.control diff --git a/build/script/opengauss_release_list_ubuntu_single b/build/script/opengauss_release_list_ubuntu_single index ae6f36ef1..dfd0958b4 100644 --- a/build/script/opengauss_release_list_ubuntu_single +++ b/build/script/opengauss_release_list_ubuntu_single @@ -61,10 +61,12 @@ ./share/postgresql/extension/security_plugin.control ./share/postgresql/extension/security_plugin--1.0.sql ./share/postgresql/extension/dolphin.control +./share/postgresql/extension/dolphin--2.0.sql ./share/postgresql/extension/dolphin--1.0.sql -./share/postgresql/extension/dolphin--1.1.sql -./share/postgresql/extension/dolphin--1.0--1.1.sql -./share/postgresql/extension/dolphin--1.1--1.0.sql +./share/postgresql/extension/dolphin--1.0--2.0.sql +./share/postgresql/extension/dolphin--2.0--1.0.sql +./share/postgresql/extension/dolphin--1.1--2.0.sql +./share/postgresql/extension/dolphin--2.0--1.1.sql ./share/postgresql/extension/openGauss_expr_dolphin.ir ./share/postgresql/extension/file_fdw--1.0.sql ./share/postgresql/extension/plpgsql.control diff --git a/build/script/x86_64_lite_list b/build/script/x86_64_lite_list index 0d1e7d927..62660db0e 100644 --- a/build/script/x86_64_lite_list +++ b/build/script/x86_64_lite_list @@ -33,10 +33,12 @@ ./share/postgresql/extension/security_plugin.control ./share/postgresql/extension/security_plugin--1.0.sql ./share/postgresql/extension/dolphin.control -./share/postgresql/extension/dolphin--1.1.sql +./share/postgresql/extension/dolphin--2.0.sql ./share/postgresql/extension/dolphin--1.0.sql -./share/postgresql/extension/dolphin--1.0--1.1.sql -./share/postgresql/extension/dolphin--1.1--1.0.sql +./share/postgresql/extension/dolphin--1.0--2.0.sql +./share/postgresql/extension/dolphin--2.0--1.0.sql +./share/postgresql/extension/dolphin--1.1--2.0.sql +./share/postgresql/extension/dolphin--2.0--1.1.sql ./share/postgresql/extension/openGauss_expr_dolphin.ir ./share/postgresql/extension/file_fdw--1.0.sql ./share/postgresql/extension/plpgsql.control diff --git a/build/script/x86_64_opengauss_list b/build/script/x86_64_opengauss_list index 5e2f16616..4923cc08a 100644 --- a/build/script/x86_64_opengauss_list +++ b/build/script/x86_64_opengauss_list @@ -69,10 +69,12 @@ ./share/postgresql/extension/ndpplugin.control ./share/postgresql/extension/ndpplugin--1.0.sql ./share/postgresql/extension/dolphin.control +./share/postgresql/extension/dolphin--2.0.sql ./share/postgresql/extension/dolphin--1.0.sql -./share/postgresql/extension/dolphin--1.1.sql -./share/postgresql/extension/dolphin--1.0--1.1.sql -./share/postgresql/extension/dolphin--1.1--1.0.sql +./share/postgresql/extension/dolphin--1.0--2.0.sql +./share/postgresql/extension/dolphin--2.0--1.0.sql +./share/postgresql/extension/dolphin--1.1--2.0.sql +./share/postgresql/extension/dolphin--2.0--1.1.sql ./share/postgresql/extension/openGauss_expr_dolphin.ir ./share/postgresql/extension/file_fdw--1.0.sql ./share/postgresql/extension/plpgsql.control diff --git a/src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback-post_catalog_maindb_92_899.sql b/src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback-post_catalog_maindb_92_899.sql index 141615579..f5b87d8e1 100644 --- a/src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback-post_catalog_maindb_92_899.sql +++ b/src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback-post_catalog_maindb_92_899.sql @@ -2,10 +2,47 @@ do $$ DECLARE ans boolean; BEGIN - for ans in select case when count(*)=1 then true else false end as ans from (select extname from pg_extension where extname='dolphin' and extversion = '1.1') + for ans in select case when count(*)=1 then true else false end as ans from (select extname from pg_extension where extname='dolphin') LOOP if ans = true then - ALTER EXTENSION dolphin UPDATE TO '1.0'; + if working_version_num() < 92850 then + ALTER EXTENSION dolphin UPDATE TO '1.0'; + elseif working_version_num() = 92850 then + ALTER EXTENSION dolphin UPDATE TO '1.1'; + else + BEGIN + ALTER EXTENSION dolphin UPDATE TO '1.9'; + EXCEPTION WHEN invalid_parameter_value THEN + BEGIN + ALTER EXTENSION dolphin UPDATE TO '1.8'; + EXCEPTION WHEN invalid_parameter_value THEN + BEGIN + ALTER EXTENSION dolphin UPDATE TO '1.7'; + EXCEPTION WHEN invalid_parameter_value THEN + BEGIN + ALTER EXTENSION dolphin UPDATE TO '1.6'; + EXCEPTION WHEN invalid_parameter_value THEN + BEGIN + ALTER EXTENSION dolphin UPDATE TO '1.5'; + EXCEPTION WHEN invalid_parameter_value THEN + BEGIN + ALTER EXTENSION dolphin UPDATE TO '1.4'; + EXCEPTION WHEN invalid_parameter_value THEN + BEGIN + ALTER EXTENSION dolphin UPDATE TO '1.3'; + EXCEPTION WHEN invalid_parameter_value THEN + BEGIN + ALTER EXTENSION dolphin UPDATE TO '1.2'; + EXCEPTION WHEN invalid_parameter_value THEN + END; + END; + END; + END; + END; + END; + END; + END; + end if; end if; exit; END LOOP; diff --git a/src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback-post_catalog_otherdb_92_899.sql b/src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback-post_catalog_otherdb_92_899.sql index 141615579..f5b87d8e1 100644 --- a/src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback-post_catalog_otherdb_92_899.sql +++ b/src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback-post_catalog_otherdb_92_899.sql @@ -2,10 +2,47 @@ do $$ DECLARE ans boolean; BEGIN - for ans in select case when count(*)=1 then true else false end as ans from (select extname from pg_extension where extname='dolphin' and extversion = '1.1') + for ans in select case when count(*)=1 then true else false end as ans from (select extname from pg_extension where extname='dolphin') LOOP if ans = true then - ALTER EXTENSION dolphin UPDATE TO '1.0'; + if working_version_num() < 92850 then + ALTER EXTENSION dolphin UPDATE TO '1.0'; + elseif working_version_num() = 92850 then + ALTER EXTENSION dolphin UPDATE TO '1.1'; + else + BEGIN + ALTER EXTENSION dolphin UPDATE TO '1.9'; + EXCEPTION WHEN invalid_parameter_value THEN + BEGIN + ALTER EXTENSION dolphin UPDATE TO '1.8'; + EXCEPTION WHEN invalid_parameter_value THEN + BEGIN + ALTER EXTENSION dolphin UPDATE TO '1.7'; + EXCEPTION WHEN invalid_parameter_value THEN + BEGIN + ALTER EXTENSION dolphin UPDATE TO '1.6'; + EXCEPTION WHEN invalid_parameter_value THEN + BEGIN + ALTER EXTENSION dolphin UPDATE TO '1.5'; + EXCEPTION WHEN invalid_parameter_value THEN + BEGIN + ALTER EXTENSION dolphin UPDATE TO '1.4'; + EXCEPTION WHEN invalid_parameter_value THEN + BEGIN + ALTER EXTENSION dolphin UPDATE TO '1.3'; + EXCEPTION WHEN invalid_parameter_value THEN + BEGIN + ALTER EXTENSION dolphin UPDATE TO '1.2'; + EXCEPTION WHEN invalid_parameter_value THEN + END; + END; + END; + END; + END; + END; + END; + END; + end if; end if; exit; END LOOP; diff --git a/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_899.sql b/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_899.sql index 57c0ffa36..91eec9005 100644 --- a/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_899.sql +++ b/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_899.sql @@ -2,10 +2,10 @@ do $$ DECLARE ans boolean; BEGIN - for ans in select case when count(*)=1 then true else false end as ans from (select extname from pg_extension where extname='dolphin' and extversion = '1.0') + for ans in select case when count(*)=1 then true else false end as ans from (select extname from pg_extension where extname='dolphin') LOOP if ans = true then - ALTER EXTENSION dolphin UPDATE TO '1.1'; + ALTER EXTENSION dolphin UPDATE TO '2.0'; end if; exit; END LOOP; diff --git a/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_899.sql b/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_899.sql index 57c0ffa36..91eec9005 100644 --- a/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_899.sql +++ b/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_899.sql @@ -2,10 +2,10 @@ do $$ DECLARE ans boolean; BEGIN - for ans in select case when count(*)=1 then true else false end as ans from (select extname from pg_extension where extname='dolphin' and extversion = '1.0') + for ans in select case when count(*)=1 then true else false end as ans from (select extname from pg_extension where extname='dolphin') LOOP if ans = true then - ALTER EXTENSION dolphin UPDATE TO '1.1'; + ALTER EXTENSION dolphin UPDATE TO '2.0'; end if; exit; END LOOP; From 2359fbe8f1c4addbaba2feda30125fb8be5fcb2a Mon Sep 17 00:00:00 2001 From: chendong76 <1209756284@qq.com> Date: Wed, 19 Jul 2023 22:37:33 +0800 Subject: [PATCH 039/304] =?UTF-8?q?1.=E5=BC=80=E5=90=AF=E6=8C=89=E9=9C=80?= =?UTF-8?q?=E5=9B=9E=E6=94=BE=E6=97=B6=EF=BC=8C=E6=8C=89=E9=9C=80=E5=9B=9E?= =?UTF-8?q?=E6=94=BE=E9=98=B6=E6=AE=B5=E4=BB=8E=E7=A3=81=E7=9B=98=E4=B8=AD?= =?UTF-8?q?=E8=AF=BB=E5=8F=96xlog=E6=97=A5=E5=BF=97=EF=BC=8C=E9=99=8D?= =?UTF-8?q?=E4=BD=8E=E5=86=85=E5=AD=98=E6=B6=88=E8=80=97=202.=E8=A7=A3?= =?UTF-8?q?=E5=86=B3=E6=8C=89=E9=9C=80=E5=9B=9E=E6=94=BE=E5=86=85=E5=AD=98?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E8=BE=83=E5=B0=8F=E6=97=B6=EF=BC=8CHashMap?= =?UTF-8?q?=E6=9E=84=E5=BB=BA=E5=8D=A1=E4=BD=8F=E7=9A=84=E9=97=AE=E9=A2=98?= =?UTF-8?q?=203.=E8=A7=84=E8=8C=83=E6=8C=89=E9=9C=80=E5=9B=9E=E6=94=BE?= =?UTF-8?q?=E4=B8=8E=E6=8C=89=E9=9C=80=E5=9B=9E=E6=94=BE=E7=9A=84=E6=97=A5?= =?UTF-8?q?=E5=BF=97=E5=9B=9E=E6=94=BE=E9=98=B6=E6=AE=B5=E5=8F=98=E9=87=8F?= =?UTF-8?q?=E5=91=BD=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/pg_controldata/pg_controldata.cpp | 4 +- src/common/backend/parser/analyze.cpp | 2 +- .../ddes/adapter/ss_dms_bufmgr.cpp | 4 +- .../ddes/adapter/ss_reform_common.cpp | 11 +- src/gausskernel/process/tcop/utility.cpp | 2 +- .../process/threadpool/knl_thread.cpp | 1 + .../ondemand_extreme_rto/dispatcher.cpp | 11 +- .../ondemand_extreme_rto/page_redo.cpp | 68 ++++--- .../ondemand_extreme_rto/redo_utils.cpp | 167 +++++++++++++++--- .../ondemand_extreme_rto/xlog_read.cpp | 109 +++++++++++- .../storage/access/transam/xlog.cpp | 6 +- .../storage/access/transam/xlogutils.cpp | 2 +- src/gausskernel/storage/ipc/ipci.cpp | 1 + src/gausskernel/storage/lmgr/lwlocknames.txt | 3 +- .../access/ondemand_extreme_rto/dispatcher.h | 1 + .../access/ondemand_extreme_rto/page_redo.h | 3 +- .../access/ondemand_extreme_rto/redo_utils.h | 3 + .../access/ondemand_extreme_rto/xlog_read.h | 15 ++ src/include/access/xlog.h | 2 +- src/include/access/xlogproc.h | 1 + src/include/ddes/dms/ss_common_attr.h | 6 +- src/include/ddes/dms/ss_dms_recovery.h | 6 +- src/include/knl/knl_thread.h | 1 + src/include/miscadmin.h | 2 +- 24 files changed, 332 insertions(+), 99 deletions(-) diff --git a/src/bin/pg_controldata/pg_controldata.cpp b/src/bin/pg_controldata/pg_controldata.cpp index e157fe200..f8fe52c25 100644 --- a/src/bin/pg_controldata/pg_controldata.cpp +++ b/src/bin/pg_controldata/pg_controldata.cpp @@ -88,8 +88,8 @@ static const char* SSClusterState(SSGlobalClusterState state) { switch (state) { case CLUSTER_IN_ONDEMAND_BUILD: return _("in on-demand build"); - case CLUSTER_IN_ONDEMAND_RECOVERY: - return _("in on-demand recovery"); + case CLUSTER_IN_ONDEMAND_REDO: + return _("in on-demand redo"); case CLUSTER_NORMAL: return _("normal"); default: diff --git a/src/common/backend/parser/analyze.cpp b/src/common/backend/parser/analyze.cpp index 1a9b8f89e..123332e80 100644 --- a/src/common/backend/parser/analyze.cpp +++ b/src/common/backend/parser/analyze.cpp @@ -615,7 +615,7 @@ Query* transformStmt(ParseState* pstate, Node* parseTree, bool isFirstNode, bool result->rightRefState = nullptr; } - PreventCommandDuringSSOndemandRecovery(parseTree); + PreventCommandDuringSSOndemandRedo(parseTree); return result; } diff --git a/src/gausskernel/ddes/adapter/ss_dms_bufmgr.cpp b/src/gausskernel/ddes/adapter/ss_dms_bufmgr.cpp index 0cf347208..1cc615efe 100644 --- a/src/gausskernel/ddes/adapter/ss_dms_bufmgr.cpp +++ b/src/gausskernel/ddes/adapter/ss_dms_bufmgr.cpp @@ -504,7 +504,7 @@ Buffer DmsReadPage(Buffer buffer, LWLockMode mode, ReadBufferMode read_mode, boo bool SSOndemandRequestPrimaryRedo(BufferTag tag) { dms_context_t dms_ctx; - int32 redo_status = ONDEMAND_REDO_INVALID; + int32 redo_status = ONDEMAND_REDO_TIMEOUT; if (!SS_STANDBY_ONDEMAND_RECOVERY) { return true; @@ -522,7 +522,7 @@ bool SSOndemandRequestPrimaryRedo(BufferTag tag) ereport(LOG, (errmodule(MOD_DMS), errmsg("[on-demand] request primary node redo page failed, page id [%d/%d/%d/%d/%d %d-%d], " - "redo statu %d", tag.rnode.spcNode, tag.rnode.dbNode, tag.rnode.relNode, (int)tag.rnode.bucketNode, + "redo status %d", tag.rnode.spcNode, tag.rnode.dbNode, tag.rnode.relNode, (int)tag.rnode.bucketNode, (int)tag.rnode.opt, tag.forkNum, tag.blockNum, redo_status))); return false; } diff --git a/src/gausskernel/ddes/adapter/ss_reform_common.cpp b/src/gausskernel/ddes/adapter/ss_reform_common.cpp index 1d64bb661..d0dee7cf1 100644 --- a/src/gausskernel/ddes/adapter/ss_reform_common.cpp +++ b/src/gausskernel/ddes/adapter/ss_reform_common.cpp @@ -37,15 +37,6 @@ #include "storage/smgr/segment_internal.h" #include "replication/walreceiver.h" -/* - * Add xlog reader private structure for page read. - */ -typedef struct XLogPageReadPrivate { - int emode; - bool fetching_ckpt; /* are we fetching a checkpoint record? */ - bool randAccess; -} XLogPageReadPrivate; - int SSXLogFileReadAnyTLI(XLogSegNo segno, int emode, uint32 sources, char* xlog_path) { char path[MAXPGPATH]; @@ -87,7 +78,7 @@ int SSXLogFileReadAnyTLI(XLogSegNo segno, int emode, uint32 sources, char* xlog_ } /* Couldn't find it. For simplicity, complain about front timeline */ - errorno = snprintf_s(path, MAXPGPATH, MAXPGPATH - 1, "%s/%08X%08X%08X", SS_XLOGDIR, + errorno = snprintf_s(path, MAXPGPATH, MAXPGPATH - 1, "%s/%08X%08X%08X", xlog_path, t_thrd.xlog_cxt.recoveryTargetTLI, (uint32)((segno) / XLogSegmentsPerXLogId), (uint32)((segno) % XLogSegmentsPerXLogId)); securec_check_ss(errorno, "", ""); diff --git a/src/gausskernel/process/tcop/utility.cpp b/src/gausskernel/process/tcop/utility.cpp index b4f3303f9..05d22c428 100755 --- a/src/gausskernel/process/tcop/utility.cpp +++ b/src/gausskernel/process/tcop/utility.cpp @@ -643,7 +643,7 @@ void PreventCommandDuringRecovery(const char* cmd_name) errmsg("cannot execute %s during recovery", cmd_name))); } -void PreventCommandDuringSSOndemandRecovery(Node* parseTree) +void PreventCommandDuringSSOndemandRedo(Node* parseTree) { switch(nodeTag(parseTree)) { case T_InsertStmt: diff --git a/src/gausskernel/process/threadpool/knl_thread.cpp b/src/gausskernel/process/threadpool/knl_thread.cpp index af937063b..8bcf7571a 100755 --- a/src/gausskernel/process/threadpool/knl_thread.cpp +++ b/src/gausskernel/process/threadpool/knl_thread.cpp @@ -1429,6 +1429,7 @@ static void knl_t_storage_init(knl_t_storage_context* storage_cxt) storage_cxt->timeoutRemoteOpera = 0; storage_cxt->dmsBufCtl = NULL; storage_cxt->ondemandXLogMem = NULL; + storage_cxt->ondemandXLogFileIdCache = NULL; } static void knl_t_port_init(knl_t_port_context* port_cxt) diff --git a/src/gausskernel/storage/access/transam/ondemand_extreme_rto/dispatcher.cpp b/src/gausskernel/storage/access/transam/ondemand_extreme_rto/dispatcher.cpp index c5bcc4e0a..cab1ab125 100644 --- a/src/gausskernel/storage/access/transam/ondemand_extreme_rto/dispatcher.cpp +++ b/src/gausskernel/storage/access/transam/ondemand_extreme_rto/dispatcher.cpp @@ -62,6 +62,7 @@ #include "access/ondemand_extreme_rto/spsc_blocking_queue.h" #include "access/ondemand_extreme_rto/redo_item.h" #include "access/ondemand_extreme_rto/batch_redo.h" +#include "access/ondemand_extreme_rto/xlog_read.h" #include "catalog/storage.h" #include @@ -98,7 +99,7 @@ static const int XLOG_INFO_SHIFT_SIZE = 4; /* xlog info flag shift size */ static const int32 MAX_PENDING = 1; static const int32 MAX_PENDING_STANDBY = 1; -static const int32 ITEM_QUQUE_SIZE_RATIO = 5; +static const int32 ITEM_QUQUE_SIZE_RATIO = 1; static const uint32 EXIT_WAIT_DELAY = 100; /* 100 us */ uint32 g_readManagerTriggerFlag = TRIGGER_NORMAL; @@ -439,9 +440,10 @@ void StartRecoveryWorkers(XLogReaderState *xlogreader, uint32 privateLen) ALLOCSET_DEFAULT_MAXSIZE, SHARED_CONTEXT); g_instance.comm_cxt.predo_cxt.redoItemHash = PRRedoItemHashInitialize(g_instance.comm_cxt.redoItemCtx); + g_dispatcher->maxItemNum = (get_batch_redo_num() + 4) * PAGE_WORK_QUEUE_SIZE * + ITEM_QUQUE_SIZE_RATIO; // 4: a startup, readmanager, txnmanager, txnworker uint32 maxParseBufNum = (uint32)((uint64)g_instance.attr.attr_storage.dms_attr.ondemand_recovery_mem_size * 1024 / (sizeof(XLogRecParseState) + sizeof(ParseBufferDesc) + sizeof(RedoMemSlot))); - g_dispatcher->maxItemNum = 4 * PAGE_WORK_QUEUE_SIZE * ITEM_QUQUE_SIZE_RATIO + maxParseBufNum; XLogParseBufferInitFunc(&(g_dispatcher->parseManager), maxParseBufNum, &recordRefOperate, RedoInterruptCallBack); /* alloc for record readbuf */ SSAllocRecordReadBuffer(xlogreader, privateLen); @@ -663,6 +665,7 @@ static void StopRecoveryWorkers(int code, Datum arg) pg_atomic_write_u32(&g_dispatcher->rtoXlogBufState.readWorkerState, WORKER_STATE_EXIT); ShutdownWalRcv(); + CloseAllXlogFileInFdCache(); FreeAllocatedRedoItem(); SSDestroyRecoveryWorkers(); g_startupTriggerState = TRIGGER_NORMAL; @@ -1910,7 +1913,7 @@ void WaitRedoFinish() SpinLockRelease(&t_thrd.shemem_ptr_cxt.XLogCtl->info_lck); /* for other nodes in cluster */ - g_instance.dms_cxt.SSReformerControl.clusterStatus = CLUSTER_IN_ONDEMAND_RECOVERY; + g_instance.dms_cxt.SSReformerControl.clusterStatus = CLUSTER_IN_ONDEMAND_REDO; SSSaveReformerCtrl(); #ifdef USE_ASSERT_CHECKING @@ -1931,7 +1934,7 @@ void WaitRedoFinish() AllItemCheck(); #endif SpinLockAcquire(&t_thrd.shemem_ptr_cxt.XLogCtl->info_lck); - t_thrd.shemem_ptr_cxt.XLogCtl->IsOnDemandRecoveryDone = true; + t_thrd.shemem_ptr_cxt.XLogCtl->IsOnDemandRedoDone = true; SpinLockRelease(&t_thrd.shemem_ptr_cxt.XLogCtl->info_lck); } diff --git a/src/gausskernel/storage/access/transam/ondemand_extreme_rto/page_redo.cpp b/src/gausskernel/storage/access/transam/ondemand_extreme_rto/page_redo.cpp index 4b5a6415a..51e880f1f 100644 --- a/src/gausskernel/storage/access/transam/ondemand_extreme_rto/page_redo.cpp +++ b/src/gausskernel/storage/access/transam/ondemand_extreme_rto/page_redo.cpp @@ -109,6 +109,7 @@ RedoItem g_GlobalLsnForwarder; RedoItem g_cleanupMark; RedoItem g_closefdMark; RedoItem g_cleanInvalidPageMark; +RedoItem g_forceDistributeMark; static const int PAGE_REDO_WORKER_ARG = 3; static const int REDO_SLEEP_50US = 50; @@ -553,6 +554,22 @@ void WaitAllRedoWorkerQueueEmpty() } } +bool OndemandXLogParseMemApproachLimit() +{ + float4 ratio = (float4)g_dispatcher->parseManager.memctl.usedblknum / g_dispatcher->parseManager.memctl.totalblknum; + if (ratio > ONDEMAND_DISTRIBUTE_RATIO) { + return true; + } + return false; +} + +void BatchRedoSendDistributeMarkToPageRedoManager(RedoItem *distributeMark) +{ + PageRedoPipeline *myRedoLine = &g_dispatcher->pageLines[g_redoWorker->slotId]; + AddPageRedoItem(myRedoLine->managerThd, distributeMark); + pg_usleep(1000000); // 1 sec +} + bool BatchRedoDistributeItems(void **eleArry, uint32 eleNum) { bool parsecomplete = false; @@ -568,6 +585,9 @@ bool BatchRedoDistributeItems(void **eleArry, uint32 eleNum) } else if (eleArry[i] == (void *)&g_cleanInvalidPageMark) { forget_range_invalid_pages((void *)eleArry[i]); } else { + if (OndemandXLogParseMemApproachLimit()) { + BatchRedoSendDistributeMarkToPageRedoManager(&g_forceDistributeMark); + } GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_3]); RedoItem *item = (RedoItem *)eleArry[i]; UpdateRecordGlobals(item, g_redoWorker->standbyState); @@ -723,7 +743,6 @@ void RedoPageManagerDistributeToRedoThd(PageRedoPipeline *myRedoLine, void RedoPageManagerDistributeBlockRecord(HTAB *redoItemHash, XLogRecParseState *parsestate) { - static uint32 total_count = 0; PageRedoPipeline *myRedoLine = &g_dispatcher->pageLines[g_redoWorker->slotId]; const uint32 WorkerNumPerMng = myRedoLine->redoThdNum; HASH_SEQ_STATUS status; @@ -731,7 +750,6 @@ void RedoPageManagerDistributeBlockRecord(HTAB *redoItemHash, XLogRecParseState HTAB *curMap = redoItemHash; hash_seq_init(&status, curMap); - total_count++; while ((redoItemEntry = (RedoItemHashEntry *)hash_seq_search(&status)) != NULL) { uint32 workId = GetWorkerId(&redoItemEntry->redoItemTag, WorkerNumPerMng); ReleaseRecParseState(myRedoLine, curMap, redoItemEntry, workId); @@ -1053,19 +1071,14 @@ static void WaitNextBarrier(XLogRecParseState *parseState) static void OnDemandPageManagerRedoSegParseState(XLogRecParseState *preState) { - static uint32 seg_total_count = 0; - static uint32 seg_full_count = 0; - Assert(g_redoWorker->slotId == 0); switch (preState->blockparse.blockhead.block_valid) { case BLOCK_DATA_SEG_EXTEND: - seg_total_count++; GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_4]); OnDemandPageManagerProcSegPipeLineSyncState(preState); CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_4]); break; case BLOCK_DATA_SEG_FULL_SYNC_TYPE: - seg_full_count++; GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_8]); OnDemandPageManagerProcSegFullSyncState(preState); CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_8]); @@ -1082,6 +1095,7 @@ static void OnDemandPageManagerRedoSegParseState(XLogRecParseState *preState) void PageManagerRedoParseState(XLogRecParseState *preState) { HTAB *hashMap = g_instance.comm_cxt.predo_cxt.redoItemHash[g_redoWorker->slotId]; + RedoItem *item = GetRedoItemPtr((XLogReaderState *)preState->refrecord); switch (preState->blockparse.blockhead.block_valid) { case BLOCK_DATA_MAIN_DATA_TYPE: @@ -1092,6 +1106,11 @@ void PageManagerRedoParseState(XLogRecParseState *preState) PRTrackAddBlock(preState, hashMap); SetCompletedReadEndPtr(g_redoWorker, preState->blockparse.blockhead.start_ptr, preState->blockparse.blockhead.end_ptr); +#ifdef USE_ASSERT_CHECKING + DoRecordCheck(preState, InvalidXLogRecPtr, false); +#endif + DereferenceRedoItem(item); // for less ondmeand recovery memory consume + preState->refrecord = NULL; CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_3]); break; case BLOCK_DATA_DDL_TYPE: @@ -1176,6 +1195,16 @@ bool PageManagerRedoDistributeItems(void **eleArry, uint32 eleNum) } else if (eleArry[i] == (void *)&g_cleanInvalidPageMark) { forget_range_invalid_pages((void *)eleArry[i]); continue; + } else if (eleArry[i] == (void *)&g_forceDistributeMark) { + // double check + if (OndemandXLogParseMemApproachLimit()) { + RedoPageManagerDistributeBlockRecord(hashMap, NULL); + ereport(WARNING, (errcode(ERRCODE_LOG), + errmsg("[On-demand] Parse buffer num approach critical value, distribute block record by force," + " slotid %d, usedblknum %d, totalblknum %d", g_redoWorker->slotId, + g_dispatcher->parseManager.memctl.usedblknum, g_dispatcher->parseManager.memctl.totalblknum))); + } + continue; } XLogRecParseState *recordblockstate = (XLogRecParseState *)eleArry[i]; XLogRecParseState *nextState = recordblockstate; @@ -1196,19 +1225,6 @@ bool PageManagerRedoDistributeItems(void **eleArry, uint32 eleNum) } while (nextState != NULL); } - float4 ratio = g_dispatcher->parseManager.memctl.usedblknum / g_dispatcher->parseManager.memctl.totalblknum; - while (ratio > ONDEMAND_DISTRIBUTE_RATIO) { - ereport(WARNING, (errcode(ERRCODE_LOG), - errmsg("[On-demand] Parse buffer num approach critical value, distribute block record by force," - " slotid %d, usedblknum %d, totalblknum %d", g_redoWorker->slotId, - g_dispatcher->parseManager.memctl.usedblknum, g_dispatcher->parseManager.memctl.totalblknum))); - GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_9]); - RedoPageManagerDistributeBlockRecord(hashMap, NULL); - CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_9]); - pg_usleep(1000000); /* 1 sec */ - ratio = g_dispatcher->parseManager.memctl.usedblknum / g_dispatcher->parseManager.memctl.totalblknum; - } - return false; } @@ -1608,12 +1624,12 @@ void RedoPageWorkerMain() bool needRelease = true; XLogRecParseState *procState = redoblockstateHead; + XLogRecParseState *reloadBlockState = NULL; Assert(procState->distributeStatus != XLOG_NO_DISTRIBUTE); MemoryContext oldCtx = MemoryContextSwitchTo(g_redoWorker->oldCtx); while (procState != NULL) { XLogRecParseState *redoblockstate = procState; - g_redoWorker->curRedoBlockState = (XLogBlockDataParse*)(&redoblockstate->blockparse.extra_rec); // nextrecord will be redo in backwards position procState = (procState->distributeStatus == XLOG_TAIL_DISTRIBUTE) ? NULL : (XLogRecParseState *)procState->nextrecord; @@ -1624,8 +1640,11 @@ void RedoPageWorkerMain() case BLOCK_DATA_FSM_TYPE: needRelease = false; GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_3]); - notfound = XLogBlockRedoForExtremeRTO(redoblockstate, &bufferinfo, notfound, + // reload from disk, because RedoPageManager already release refrecord in on-demand build stage + reloadBlockState = OndemandRedoReloadXLogRecord(redoblockstate); + notfound = XLogBlockRedoForExtremeRTO(reloadBlockState, &bufferinfo, notfound, g_redoWorker->timeCostList[TIME_COST_STEP_4], g_redoWorker->timeCostList[TIME_COST_STEP_5]); + OndemandRedoReleaseXLogRecord(reloadBlockState); DereferenceRecParseState(redoblockstate); SetCompletedReadEndPtr(g_redoWorker, redoblockstate->blockparse.blockhead.start_ptr, redoblockstate->blockparse.blockhead.end_ptr); @@ -1754,10 +1773,6 @@ static inline bool ReadPageWorkerStop() void PushToWorkerLsn() { - static uint32 cur_recor_count = 0; - - cur_recor_count++; - if (!IsExtremeRtoRunning()) { return; } @@ -1767,7 +1782,6 @@ void PushToWorkerLsn() refCount = pg_atomic_read_u32(&g_GlobalLsnForwarder.record.refcount); RedoInterruptCallBack(); } while (refCount != 0 && !ReadPageWorkerStop()); - cur_recor_count = 0; SendLsnFowarder(); } diff --git a/src/gausskernel/storage/access/transam/ondemand_extreme_rto/redo_utils.cpp b/src/gausskernel/storage/access/transam/ondemand_extreme_rto/redo_utils.cpp index 7ba976b25..c93e275f0 100644 --- a/src/gausskernel/storage/access/transam/ondemand_extreme_rto/redo_utils.cpp +++ b/src/gausskernel/storage/access/transam/ondemand_extreme_rto/redo_utils.cpp @@ -25,8 +25,17 @@ #include "access/ondemand_extreme_rto/batch_redo.h" #include "access/ondemand_extreme_rto/dispatcher.h" #include "access/ondemand_extreme_rto/redo_utils.h" +#include "access/ondemand_extreme_rto/xlog_read.h" #include "storage/lock/lwlock.h" +/* + * Add xlog reader private structure for page read. + */ +typedef struct XLogPageReadPrivate { + const char* datadir; + TimeLineID tli; +} XLogPageReadPrivate; + Size OndemandRecoveryShmemSize(void) { Size size = 0; @@ -48,6 +57,22 @@ void OndemandRecoveryShmemInit(void) } } +void OndemandXlogFileIdCacheInit(void) +{ + HASHCTL ctl; + + /* hash accessed by database file id */ + errno_t rc = memset_s(&ctl, sizeof(ctl), 0, sizeof(ctl)); + securec_check(rc, "", ""); + ctl.keysize = sizeof(XLogFileId); + ctl.entrysize = sizeof(XLogFileIdCacheEntry); + ctl.hash = tag_hash; + t_thrd.storage_cxt.ondemandXLogFileIdCache = hash_create("Ondemand extreme rto xlogfile handle cache", 8, &ctl, + HASH_ELEM | HASH_FUNCTION | HASH_SHRCTX); + if (!t_thrd.storage_cxt.ondemandXLogFileIdCache) + ereport(FATAL, (errmsg("could not initialize ondemand xlogfile handle hash table"))); +} + /* add for batch redo mem manager */ void *OndemandXLogMemCtlInit(RedoMemManager *memctl, Size itemsize, int itemnum) { @@ -73,7 +98,7 @@ RedoMemSlot *OndemandXLogMemAlloc(RedoMemManager *memctl) { RedoMemSlot *nextfreeslot = NULL; do { - LWLockAcquire(OndemandXlogMemAllocLock, LW_EXCLUSIVE); + LWLockAcquire(OndemandXLogMemAllocLock, LW_EXCLUSIVE); if (memctl->firstfreeslot == InvalidBuffer) { memctl->firstfreeslot = AtomicExchangeBuffer(&memctl->firstreleaseslot, InvalidBuffer); pg_read_barrier(); @@ -85,7 +110,7 @@ RedoMemSlot *OndemandXLogMemAlloc(RedoMemManager *memctl) memctl->usedblknum++; nextfreeslot->freeNext = InvalidBuffer; } - LWLockRelease(OndemandXlogMemAllocLock); + LWLockRelease(OndemandXLogMemAllocLock); if (memctl->doInterrupt != NULL) { memctl->doInterrupt(); @@ -106,14 +131,14 @@ void OndemandXLogMemRelease(RedoMemManager *memctl, Buffer bufferid) } bufferslot = &(memctl->memslot[bufferid - 1]); Assert(bufferslot->freeNext == InvalidBuffer); - LWLockAcquire(OndemandXlogMemAllocLock, LW_EXCLUSIVE); + LWLockAcquire(OndemandXLogMemAllocLock, LW_EXCLUSIVE); Buffer oldFirst = AtomicReadBuffer(&memctl->firstreleaseslot); pg_memory_barrier(); do { AtomicWriteBuffer(&bufferslot->freeNext, oldFirst); } while (!AtomicCompareExchangeBuffer(&memctl->firstreleaseslot, &oldFirst, bufferid)); memctl->usedblknum--; - LWLockRelease(OndemandXlogMemAllocLock); + LWLockRelease(OndemandXLogMemAllocLock); } @@ -142,46 +167,63 @@ void OndemandXLogParseBufferDestory(RedoParseManager *parsemanager) XLogRecParseState *OndemandXLogParseBufferAllocList(RedoParseManager *parsemanager, XLogRecParseState *blkstatehead, void *record) { - RedoMemManager *memctl = &(parsemanager->memctl); - RedoMemSlot *allocslot = NULL; - ParseBufferDesc *descstate = NULL; XLogRecParseState *recordstate = NULL; - allocslot = OndemandXLogMemAlloc(memctl); - if (allocslot == NULL) { - ereport(PANIC, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), - errmsg("XLogParseBufferAlloc Allocated buffer failed!, taoalblknum:%u, usedblknum:%u", - memctl->totalblknum, memctl->usedblknum))); - return NULL; + if (parsemanager == NULL) { + recordstate = (XLogRecParseState*)palloc(sizeof(XLogRecParseState)); + errno_t rc = memset_s((void*)recordstate, sizeof(XLogRecParseState), 0, sizeof(XLogRecParseState)); + securec_check(rc, "\0", "\0"); + recordstate->manager = &(ondemand_extreme_rto::g_dispatcher->parseManager); + recordstate->distributeStatus = XLOG_SKIP_DISTRIBUTE; + } else { + RedoMemManager *memctl = &(parsemanager->memctl); + RedoMemSlot *allocslot = NULL; + ParseBufferDesc *descstate = NULL; + + allocslot = OndemandXLogMemAlloc(memctl); + if (allocslot == NULL) { + ereport(PANIC, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), + errmsg("XLogParseBufferAlloc Allocated buffer failed!, taoalblknum:%u, usedblknum:%u", + memctl->totalblknum, memctl->usedblknum))); + return NULL; + } + + pg_read_barrier(); + Assert(allocslot->buf_id != InvalidBuffer); + Assert(memctl->itemsize == (sizeof(XLogRecParseState) + sizeof(ParseBufferDesc))); + descstate = (ParseBufferDesc *)((char *)parsemanager->parsebuffers + memctl->itemsize * (allocslot->buf_id - 1)); + descstate->buff_id = allocslot->buf_id; + Assert(descstate->state == 0); + descstate->state = 1; + descstate->refcount = 0; + recordstate = (XLogRecParseState *)((char *)descstate + sizeof(ParseBufferDesc)); + recordstate->manager = parsemanager; + recordstate->distributeStatus = XLOG_NO_DISTRIBUTE; + + if (parsemanager->refOperate != NULL) { + parsemanager->refOperate->refCount(record); + } } - pg_read_barrier(); - Assert(allocslot->buf_id != InvalidBuffer); - Assert(memctl->itemsize == (sizeof(XLogRecParseState) + sizeof(ParseBufferDesc))); - descstate = (ParseBufferDesc *)((char *)parsemanager->parsebuffers + memctl->itemsize * (allocslot->buf_id - 1)); - descstate->buff_id = allocslot->buf_id; - Assert(descstate->state == 0); - descstate->state = 1; - descstate->refcount = 0; - recordstate = (XLogRecParseState *)((char *)descstate + sizeof(ParseBufferDesc)); recordstate->nextrecord = NULL; - recordstate->manager = parsemanager; recordstate->refrecord = record; recordstate->isFullSync = false; - recordstate->distributeStatus = XLOG_NO_DISTRIBUTE; if (blkstatehead != NULL) { recordstate->nextrecord = blkstatehead->nextrecord; blkstatehead->nextrecord = (void *)recordstate; } - if (parsemanager->refOperate != NULL) - parsemanager->refOperate->refCount(record); - return recordstate; } void OndemandXLogParseBufferRelease(XLogRecParseState *recordstate) { + if (recordstate->distributeStatus == XLOG_SKIP_DISTRIBUTE) { + // alloc in pageRedoWorker or backends + pfree(recordstate); + return; + } + RedoMemManager *memctl = &(recordstate->manager->memctl); ParseBufferDesc *descstate = NULL; @@ -209,6 +251,7 @@ BufferDesc *RedoForOndemandExtremeRTOQuery(BufferDesc *bufHdr, char relpersisten ondemand_extreme_rto::RedoItemHashEntry *redoItemEntry = NULL; ondemand_extreme_rto::RedoItemTag redoItemTag; XLogRecParseState *procState = NULL; + XLogRecParseState *reloadBlockState = NULL; XLogBlockHead *procBlockHead = NULL; XLogBlockHead *blockHead = NULL; RedoBufferInfo bufferInfo; @@ -280,7 +323,10 @@ BufferDesc *RedoForOndemandExtremeRTOQuery(BufferDesc *bufHdr, char relpersisten case BLOCK_DATA_VM_TYPE: case BLOCK_DATA_FSM_TYPE: needMarkDirty = true; - XlogBlockRedoForOndemandExtremeRTOQuery(redoBlockState, &bufferInfo); + // reload from disk, because RedoPageManager already release refrecord in on-demand build stage + reloadBlockState = OndemandRedoReloadXLogRecord(redoBlockState); + XlogBlockRedoForOndemandExtremeRTOQuery(reloadBlockState, &bufferInfo); + OndemandRedoReleaseXLogRecord(reloadBlockState); break; case BLOCK_DATA_XLOG_COMMON_TYPE: case BLOCK_DATA_DDL_TYPE: @@ -307,6 +353,71 @@ BufferDesc *RedoForOndemandExtremeRTOQuery(BufferDesc *bufHdr, char relpersisten return bufHdr; } +bool IsTargetBlockState(XLogRecParseState *targetblockstate, XLogRecParseState* curblockstate) +{ + if (memcmp(&targetblockstate->blockparse.blockhead, &curblockstate->blockparse.blockhead, sizeof(XLogBlockHead)) != 0) { + return false; + } + return true; +} + +// only used in ondemand redo stage +XLogRecParseState *OndemandRedoReloadXLogRecord(XLogRecParseState *redoblockstate) +{ + uint32 blockNum = 0; + char *errormsg = NULL; + XLogRecParseState *recordBlockState = NULL; + XLogPageReadPrivate readPrivate = { + .datadir = NULL, + .tli = GetRecoveryTargetTLI() + }; + + XLogReaderState *xlogreader = XLogReaderAllocate(&SimpleXLogPageReadInFdCache, &readPrivate); // do not use pre-read + + // step1: read record + XLogRecord *record = XLogReadRecord(xlogreader, redoblockstate->blockparse.blockhead.start_ptr, &errormsg, + true, g_instance.dms_cxt.SSRecoveryInfo.recovery_xlog_dir); + if (record == NULL) { + ereport(PANIC, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), + errmsg("[On-demand] reload xlog record failed at %X/%X, errormsg: %s", + (uint32)(redoblockstate->blockparse.blockhead.start_ptr >> 32), + (uint32)redoblockstate->blockparse.blockhead.start_ptr, errormsg))); + } + + // step2: parse to block + do { + recordBlockState = XLogParseToBlockForExtermeRTO(xlogreader, &blockNum); + if (recordBlockState != NULL) { + break; + } + Assert(blockNum != 0); // out of memory + } while (true); + + // step3: find target parse state + XLogRecParseState *nextState = recordBlockState; + XLogRecParseState *targetState = NULL; + do { + XLogRecParseState *preState = nextState; + nextState = (XLogRecParseState *)nextState->nextrecord; + preState->nextrecord = NULL; + + if (IsTargetBlockState(preState, redoblockstate)) { + targetState = preState; + } else { + OndemandXLogParseBufferRelease(preState); + } + } while (nextState != NULL); + + return targetState; +} + +// only used in ondemand redo stage +void OndemandRedoReleaseXLogRecord(XLogRecParseState *reloadBlockState) +{ + XLogReaderFree((XLogReaderState*)reloadBlockState->refrecord); + OndemandXLogParseBufferRelease(reloadBlockState); +} + void OnDemandSendRecoveryEndMarkToWorkersAndWaitForReach(int code) { ondemand_extreme_rto::SendRecoveryEndMarkToWorkersAndWaitForReach(code); diff --git a/src/gausskernel/storage/access/transam/ondemand_extreme_rto/xlog_read.cpp b/src/gausskernel/storage/access/transam/ondemand_extreme_rto/xlog_read.cpp index e3c8726df..075e17027 100644 --- a/src/gausskernel/storage/access/transam/ondemand_extreme_rto/xlog_read.cpp +++ b/src/gausskernel/storage/access/transam/ondemand_extreme_rto/xlog_read.cpp @@ -23,6 +23,7 @@ #include "access/ondemand_extreme_rto/spsc_blocking_queue.h" #include "access/ondemand_extreme_rto/dispatcher.h" +#include "access/ondemand_extreme_rto/xlog_read.h" #include "access/multi_redo_api.h" #include "access/xlog.h" #include "ddes/dms/ss_reform_common.h" @@ -30,6 +31,7 @@ #include "replication/dcf_replication.h" #include "replication/shared_storage_walreceiver.h" #include "storage/ipc.h" +#include "storage/file/fio_device.h" namespace ondemand_extreme_rto { static bool DoEarlyExit() @@ -251,14 +253,7 @@ int ParallelXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, xlogreader->readBuf = g_dispatcher->rtoXlogBufState.readBuf; for (;;) { - uint32 readSource = pg_atomic_read_u32(&(g_recordbuffer->readSource)); - if (readSource & XLOG_FROM_STREAM) { - readLen = ParallelXLogReadWorkBufRead(xlogreader, targetPagePtr, reqLen, targetRecPtr, readTLI); - } else { - readLen = SSXLogPageRead(xlogreader, targetPagePtr, reqLen, targetRecPtr, xlogreader->readBuf, - readTLI, NULL); - } - + readLen = SSXLogPageRead(xlogreader, targetPagePtr, reqLen, targetRecPtr, xlogreader->readBuf, readTLI, NULL); if (readLen > 0 || t_thrd.xlog_cxt.recoveryTriggered || !t_thrd.xlog_cxt.StandbyMode || DoEarlyExit()) { return readLen; } @@ -704,4 +699,100 @@ XLogRecord *XLogParallelReadNextRecord(XLogReaderState *xlogreader) } } -} // namespace ondemand_extreme_rto \ No newline at end of file +} // namespace ondemand_extreme_rto + +typedef struct XLogPageReadPrivate { + const char *datadir; + TimeLineID tli; +} XLogPageReadPrivate; + +void InitXLogFileId(XLogRecPtr targetPagePtr, TimeLineID timeLine, XLogFileId* id) +{ + XLByteToSeg(targetPagePtr, id->segno); + id->tli = timeLine; +} + +/* XLogreader callback function, to read a WAL page */ +int SimpleXLogPageReadInFdCache(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, + char *readBuf, TimeLineID *pageTLI, char* xlog_path) +{ + XLogPageReadPrivate *readprivate = (XLogPageReadPrivate *)xlogreader->private_data; + uint32 targetPageOff; + int ss_c = 0; + char xlogfpath[MAXPGPATH]; + XLogFileId xlogfileid; + XLogFileIdCacheEntry *entry; + int xlogreadfd = -1; + bool found = false; + + InitXLogFileId(targetPagePtr, readprivate->tli, &xlogfileid); + + (void)LWLockAcquire(OndemandXLogFileHandleLock, LW_SHARED); + entry = (XLogFileIdCacheEntry *)hash_search(t_thrd.storage_cxt.ondemandXLogFileIdCache, + (void *)&xlogfileid, HASH_FIND, &found); + if (found) { + xlogreadfd = entry->fd; + } + LWLockRelease(OndemandXLogFileHandleLock); + + if (xlogreadfd == -1) { + Assert(!found); + + (void)LWLockAcquire(OndemandXLogFileHandleLock, LW_EXCLUSIVE); + entry = (XLogFileIdCacheEntry *)hash_search(t_thrd.storage_cxt.ondemandXLogFileIdCache, + (void *)&xlogfileid, HASH_ENTER, &found); + if (entry == NULL) { + report_invalid_record(xlogreader, + "SimpleXLogPageReadInFdCache could not create xlogfile handle entry \"%s\": %s\n", + xlogfpath, strerror(errno)); + LWLockRelease(OndemandXLogFileHandleLock); + return -1; + } + + if (!found) { + ss_c = snprintf_s(xlogfpath, MAXPGPATH, MAXPGPATH - 1, "%s/%08X%08X%08X", xlog_path, readprivate->tli, + (uint32)((xlogfileid.segno) / XLogSegmentsPerXLogId), + (uint32)((xlogfileid.segno) % XLogSegmentsPerXLogId)); + securec_check_ss(ss_c, "", ""); + entry->fd = open(xlogfpath, O_RDONLY | PG_BINARY, 0); + if (entry->fd < 0) { + report_invalid_record(xlogreader, "SimpleXLogPageReadInFdCache could not open file \"%s\": %s\n", + xlogfpath, strerror(errno)); + LWLockRelease(OndemandXLogFileHandleLock); + return -1; + } + } + xlogreadfd = entry->fd; + LWLockRelease(OndemandXLogFileHandleLock); + } + + /* + * At this point, we have the right segment open. + */ + Assert(xlogreadfd != -1); + + targetPageOff = targetPagePtr % XLogSegSize; + /* Read the requested page */ + if (pread(xlogreadfd, readBuf, XLOG_BLCKSZ, (off_t)targetPageOff) != XLOG_BLCKSZ) { + report_invalid_record(xlogreader, "SimpleXLogPageReadInFdCache could not pread from file \"%s\": %s\n", + xlogfpath, strerror(errno)); + return -1; + } + *pageTLI = readprivate->tli; + return XLOG_BLCKSZ; +} + +void CloseAllXlogFileInFdCache(void) +{ + HASH_SEQ_STATUS status; + XLogFileIdCacheEntry *entry = NULL; + hash_seq_init(&status, t_thrd.storage_cxt.ondemandXLogFileIdCache); + + while ((entry = (XLogFileIdCacheEntry *)hash_seq_search(&status)) != NULL) { + close(entry->fd); + if (hash_search(t_thrd.storage_cxt.ondemandXLogFileIdCache, (void *)&entry->id, HASH_REMOVE, NULL) == NULL) { + ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("[On-demand] xlogfile handle cache hash table corrupted"))); + } + } + return; +} diff --git a/src/gausskernel/storage/access/transam/xlog.cpp b/src/gausskernel/storage/access/transam/xlog.cpp index 86389e038..4cd9ef70d 100755 --- a/src/gausskernel/storage/access/transam/xlog.cpp +++ b/src/gausskernel/storage/access/transam/xlog.cpp @@ -6377,7 +6377,7 @@ void XLOGShmemInit(void) t_thrd.shemem_ptr_cxt.XLogCtl->SharedRecoveryInProgress = true; t_thrd.shemem_ptr_cxt.XLogCtl->IsRecoveryDone = false; t_thrd.shemem_ptr_cxt.XLogCtl->IsOnDemandBuildDone = false; - t_thrd.shemem_ptr_cxt.XLogCtl->IsOnDemandRecoveryDone = false; + t_thrd.shemem_ptr_cxt.XLogCtl->IsOnDemandRedoDone = false; t_thrd.shemem_ptr_cxt.XLogCtl->SharedHotStandbyActive = false; t_thrd.shemem_ptr_cxt.XLogCtl->WalWriterSleeping = false; t_thrd.shemem_ptr_cxt.XLogCtl->xlogFlushPtrForPerRead = InvalidXLogRecPtr; @@ -8501,7 +8501,7 @@ static void XLogMakeUpRemainSegsContent(char *contentBuffer) void XLogCheckRemainSegs() { - if (SS_ONDEMAND_BUILD_DONE && !SS_ONDEMAND_RECOVERY_DONE) { + if (SS_ONDEMAND_BUILD_DONE && !SS_ONDEMAND_REDO_DONE) { return; } @@ -9302,7 +9302,7 @@ void StartupXLOG(void) t_thrd.shemem_ptr_cxt.XLogCtl->ckptXid = checkPoint.oldestXid; t_thrd.shemem_ptr_cxt.XLogCtl->IsRecoveryDone = false; t_thrd.shemem_ptr_cxt.XLogCtl->IsOnDemandBuildDone = false; - t_thrd.shemem_ptr_cxt.XLogCtl->IsOnDemandRecoveryDone = false; + t_thrd.shemem_ptr_cxt.XLogCtl->IsOnDemandRedoDone = false; latestCompletedXid = checkPoint.nextXid; TransactionIdRetreat(latestCompletedXid); diff --git a/src/gausskernel/storage/access/transam/xlogutils.cpp b/src/gausskernel/storage/access/transam/xlogutils.cpp index 211d34d50..c5db30a85 100644 --- a/src/gausskernel/storage/access/transam/xlogutils.cpp +++ b/src/gausskernel/storage/access/transam/xlogutils.cpp @@ -513,7 +513,7 @@ static void CollectInvalidPagesStates(uint32 *nstates_ptr, InvalidPagesState *** /* Complain about any remaining invalid-page entries */ void XLogCheckInvalidPages(void) { - if (SS_ONDEMAND_BUILD_DONE && !SS_ONDEMAND_RECOVERY_DONE) { + if (SS_ONDEMAND_BUILD_DONE && !SS_ONDEMAND_REDO_DONE) { return; } diff --git a/src/gausskernel/storage/ipc/ipci.cpp b/src/gausskernel/storage/ipc/ipci.cpp index 607dd3435..0f1a23838 100644 --- a/src/gausskernel/storage/ipc/ipci.cpp +++ b/src/gausskernel/storage/ipc/ipci.cpp @@ -451,6 +451,7 @@ void CreateSharedMemoryAndSemaphores(bool makePrivate, int port) if (g_instance.attr.attr_storage.dms_attr.enable_ondemand_recovery) { OndemandRecoveryShmemInit(); + OndemandXlogFileIdCacheInit(); } if (g_instance.ckpt_cxt_ctl->prune_queue_lock == NULL) { diff --git a/src/gausskernel/storage/lmgr/lwlocknames.txt b/src/gausskernel/storage/lmgr/lwlocknames.txt index 3a3fe6b75..dffbc1a6e 100755 --- a/src/gausskernel/storage/lmgr/lwlocknames.txt +++ b/src/gausskernel/storage/lmgr/lwlocknames.txt @@ -138,4 +138,5 @@ GsStackLock 128 ConfigFileLock 129 DropArchiveSlotLock 130 AboCacheLock 131 -OndemandXlogMemAllocLock 132 +OndemandXLogMemAllocLock 132 +OndemandXLogFileHandleLock 133 diff --git a/src/include/access/ondemand_extreme_rto/dispatcher.h b/src/include/access/ondemand_extreme_rto/dispatcher.h index 17f9958cf..70a216f6a 100644 --- a/src/include/access/ondemand_extreme_rto/dispatcher.h +++ b/src/include/access/ondemand_extreme_rto/dispatcher.h @@ -179,6 +179,7 @@ typedef struct { extern LogDispatcher *g_dispatcher; extern RedoItem g_GlobalLsnForwarder; extern RedoItem g_cleanupMark; +extern RedoItem g_forceDistributeMark; extern THR_LOCAL RecordBufferState *g_recordbuffer; const static uint64 OUTPUT_WAIT_COUNT = 0x7FFFFFF; diff --git a/src/include/access/ondemand_extreme_rto/page_redo.h b/src/include/access/ondemand_extreme_rto/page_redo.h index 005d76a9d..285d66c2a 100644 --- a/src/include/access/ondemand_extreme_rto/page_redo.h +++ b/src/include/access/ondemand_extreme_rto/page_redo.h @@ -42,7 +42,7 @@ namespace ondemand_extreme_rto { #define ONDEMAND_DISTRIBUTE_RATIO 0.9 -static const uint32 PAGE_WORK_QUEUE_SIZE = 2097152; +static const uint32 PAGE_WORK_QUEUE_SIZE = 65536; static const uint32 ONDEMAND_EXTREME_RTO_ALIGN_LEN = 16; /* need 128-bit aligned */ static const uint32 MAX_REMOTE_READ_INFO_NUM = 100; @@ -185,7 +185,6 @@ struct PageRedoWorker { RedoBufferManager bufferManager; RedoTimeCost timeCostList[TIME_COST_NUM]; char page[BLCKSZ]; - XLogBlockDataParse *curRedoBlockState; }; diff --git a/src/include/access/ondemand_extreme_rto/redo_utils.h b/src/include/access/ondemand_extreme_rto/redo_utils.h index a5a95a2b5..6a56cb3d9 100644 --- a/src/include/access/ondemand_extreme_rto/redo_utils.h +++ b/src/include/access/ondemand_extreme_rto/redo_utils.h @@ -28,12 +28,15 @@ Size OndemandRecoveryShmemSize(void); void OndemandRecoveryShmemInit(void); +void OndemandXlogFileIdCacheInit(void); void OndemandXLogParseBufferInit(RedoParseManager *parsemanager, int buffernum, RefOperate *refOperate, InterruptFunc interruptOperte); void OndemandXLogParseBufferDestory(RedoParseManager *parsemanager); XLogRecParseState *OndemandXLogParseBufferAllocList(RedoParseManager *parsemanager, XLogRecParseState *blkstatehead, void *record); void OndemandXLogParseBufferRelease(XLogRecParseState *recordstate); +XLogRecParseState *OndemandRedoReloadXLogRecord(XLogRecParseState *redoblockstate); +void OndemandRedoReleaseXLogRecord(XLogRecParseState *reloadBlockState); void OnDemandSendRecoveryEndMarkToWorkersAndWaitForReach(int code); void OnDemandWaitRedoFinish(); diff --git a/src/include/access/ondemand_extreme_rto/xlog_read.h b/src/include/access/ondemand_extreme_rto/xlog_read.h index 6642a013e..10cb8c251 100644 --- a/src/include/access/ondemand_extreme_rto/xlog_read.h +++ b/src/include/access/ondemand_extreme_rto/xlog_read.h @@ -31,6 +31,21 @@ namespace ondemand_extreme_rto { XLogRecord* XLogParallelReadNextRecord(XLogReaderState* xlogreader); XLogRecord *ReadNextXLogRecord(XLogReaderState **xlogreaderptr, int emode); +XLogRecord *ParallelReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg); } // namespace ondemand_extreme_rto + +typedef struct XLogFileId { + XLogSegNo segno; + TimeLineID tli; +} XLogFileId; + +typedef struct XLogFileIdCacheEntry { + XLogFileId id; + int fd; +} XLogFileIdCacheEntry; + +int SimpleXLogPageReadInFdCache(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, + XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *pageTLI, char* xlog_path = NULL); +void CloseAllXlogFileInFdCache(void); #endif /* ONDEMAND_EXTREME_RTO_XLOG_READ_H */ \ No newline at end of file diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h index 66ba231bd..04fc92a38 100755 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@ -541,7 +541,7 @@ typedef struct XLogCtlData { bool IsRecoveryDone; bool IsOnDemandBuildDone; - bool IsOnDemandRecoveryDone; + bool IsOnDemandRedoDone; /* * SharedHotStandbyActive indicates if we're still in crash or archive diff --git a/src/include/access/xlogproc.h b/src/include/access/xlogproc.h index 9d76ca5fa..fba9b6e3f 100755 --- a/src/include/access/xlogproc.h +++ b/src/include/access/xlogproc.h @@ -693,6 +693,7 @@ typedef enum { XLOG_HEAD_DISTRIBUTE, XLOG_MID_DISTRIBUTE, XLOG_TAIL_DISTRIBUTE, + XLOG_SKIP_DISTRIBUTE, } XlogDistributePos; typedef struct { diff --git a/src/include/ddes/dms/ss_common_attr.h b/src/include/ddes/dms/ss_common_attr.h index c56b1978a..cefce4989 100644 --- a/src/include/ddes/dms/ss_common_attr.h +++ b/src/include/ddes/dms/ss_common_attr.h @@ -150,7 +150,7 @@ #define SS_CLUSTER_ONDEMAND_BUILD \ (ENABLE_DMS && (g_instance.dms_cxt.SSReformerControl.clusterStatus == CLUSTER_IN_ONDEMAND_BUILD)) #define SS_CLUSTER_ONDEMAND_RECOVERY \ - (ENABLE_DMS && (g_instance.dms_cxt.SSReformerControl.clusterStatus == CLUSTER_IN_ONDEMAND_RECOVERY)) + (ENABLE_DMS && (g_instance.dms_cxt.SSReformerControl.clusterStatus == CLUSTER_IN_ONDEMAND_REDO)) #define SS_CLUSTER_ONDEMAND_NORMAL \ (ENABLE_DMS && (g_instance.dms_cxt.SSReformerControl.clusterStatus == CLUSTER_NORMAL)) #define SS_STANDBY_ONDEMAND_BUILD (SS_STANDBY_MODE && SS_CLUSTER_ONDEMAND_BUILD) @@ -226,7 +226,7 @@ typedef enum SSReformType { typedef enum SSGlobalClusterState { CLUSTER_IN_ONDEMAND_BUILD = 0, - CLUSTER_IN_ONDEMAND_RECOVERY, + CLUSTER_IN_ONDEMAND_REDO, CLUSTER_NORMAL } SSGlobalClusterState; @@ -234,7 +234,7 @@ typedef enum SSOndemandRequestRedoStatus { ONDEMAND_REDO_DONE = 0, ONDEMAND_REDO_SKIP, ONDEMAND_REDO_FAIL, - ONDEMAND_REDO_INVALID + ONDEMAND_REDO_TIMEOUT } SSOndemandRequestRedoStatus; diff --git a/src/include/ddes/dms/ss_dms_recovery.h b/src/include/ddes/dms/ss_dms_recovery.h index 98d210b17..8645feef7 100644 --- a/src/include/ddes/dms/ss_dms_recovery.h +++ b/src/include/ddes/dms/ss_dms_recovery.h @@ -35,11 +35,11 @@ #define SS_IN_ONDEMAND_RECOVERY (ENABLE_DMS && g_instance.dms_cxt.SSRecoveryInfo.in_ondemand_recovery == true) #define SS_ONDEMAND_BUILD_DONE (ENABLE_DMS && SS_IN_ONDEMAND_RECOVERY \ && t_thrd.shemem_ptr_cxt.XLogCtl->IsOnDemandBuildDone == true) -#define SS_ONDEMAND_RECOVERY_DONE (ENABLE_DMS && SS_IN_ONDEMAND_RECOVERY \ - && t_thrd.shemem_ptr_cxt.XLogCtl->IsOnDemandRecoveryDone == true) +#define SS_ONDEMAND_REDO_DONE (SS_IN_ONDEMAND_RECOVERY \ + && t_thrd.shemem_ptr_cxt.XLogCtl->IsOnDemandRedoDone == true) #define SS_REPLAYED_BY_ONDEMAND (ENABLE_DMS && !SS_IN_ONDEMAND_RECOVERY && \ t_thrd.shemem_ptr_cxt.XLogCtl->IsOnDemandBuildDone == true && \ - t_thrd.shemem_ptr_cxt.XLogCtl->IsOnDemandRecoveryDone == true) + t_thrd.shemem_ptr_cxt.XLogCtl->IsOnDemandRedoDone == true) #define REFORM_CTRL_VERSION 1 diff --git a/src/include/knl/knl_thread.h b/src/include/knl/knl_thread.h index f58b2a0ca..524c1524f 100755 --- a/src/include/knl/knl_thread.h +++ b/src/include/knl/knl_thread.h @@ -2806,6 +2806,7 @@ typedef struct knl_t_storage_context { char* PcaBufferBlocks; dms_buf_ctrl_t* dmsBufCtl; char* ondemandXLogMem; + struct HTAB* ondemandXLogFileIdCache; } knl_t_storage_context; typedef struct knl_t_port_context { diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h index 5dce1abc4..9e4c92767 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h @@ -416,7 +416,7 @@ extern bool stack_is_too_deep(void); /* in tcop/utility.c */ extern void PreventCommandIfReadOnly(const char* cmdname); extern void PreventCommandDuringRecovery(const char* cmdname); -extern void PreventCommandDuringSSOndemandRecovery(Node* parseTree); +extern void PreventCommandDuringSSOndemandRedo(Node* parseTree); extern int trace_recovery(int trace_level); From da94b2b95a97edd811d13708f05ce32246d54a96 Mon Sep 17 00:00:00 2001 From: gbzhangkai Date: Fri, 21 Jul 2023 10:38:45 +0800 Subject: [PATCH 040/304] =?UTF-8?q?=E5=B7=A5=E5=85=B7=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=EF=BC=88bin=EF=BC=89=E4=B8=AD=E6=97=A0=E7=AC=A6=E5=8F=B7?= =?UTF-8?q?=E7=B1=BB=E5=9E=8B=E6=AF=94=E8=BE=83=E4=B8=8D=E5=BD=93=E5=AF=BC?= =?UTF-8?q?=E8=87=B4=E5=88=A4=E6=96=AD=E6=9D=A1=E4=BB=B6=E5=86=97=E4=BD=99?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/gs_cgroup/cgexcp.cpp | 2 +- src/bin/gs_guc/cluster_config.cpp | 2 +- src/bin/gs_loader/gs_loader.cpp | 2 +- src/bin/pg_basebackup/pg_basebackup.cpp | 2 +- src/bin/pg_ctl/backup.cpp | 2 +- src/bin/pg_dump/pg_backup_tar.cpp | 2 +- src/bin/pg_probackup/catalog.cpp | 8 ++++---- src/bin/pg_probackup/delete.cpp | 19 ++++++++----------- src/bin/pg_probackup/logger.cpp | 2 +- 9 files changed, 19 insertions(+), 22 deletions(-) diff --git a/src/bin/gs_cgroup/cgexcp.cpp b/src/bin/gs_cgroup/cgexcp.cpp index 585d67302..2c21447a6 100644 --- a/src/bin/gs_cgroup/cgexcp.cpp +++ b/src/bin/gs_cgroup/cgexcp.cpp @@ -54,7 +54,7 @@ */ static int cgexcp_skewpercent_is_invalid(const except_data_t* except) { - if ((except->skewpercent > 0 && except->qualitime > 0) || (except->skewpercent <= 0 && except->qualitime <= 0)) + if ((except->skewpercent > 0 && except->qualitime > 0) || (except->skewpercent == 0 && except->qualitime == 0)) return 0; return 1; diff --git a/src/bin/gs_guc/cluster_config.cpp b/src/bin/gs_guc/cluster_config.cpp index 77b83c08e..58ac4ee6b 100644 --- a/src/bin/gs_guc/cluster_config.cpp +++ b/src/bin/gs_guc/cluster_config.cpp @@ -684,7 +684,7 @@ int init_gauss_cluster_config(void) return 1; } - if (g_nodeHeader.node <= 0) { + if (g_nodeHeader.node == 0) { write_stderr("ERROR: Invalid cluster_staic_config file," " curerent node id is:%d .\n", (int32)g_nodeHeader.node); diff --git a/src/bin/gs_loader/gs_loader.cpp b/src/bin/gs_loader/gs_loader.cpp index fd3e8b07c..2a8398652 100644 --- a/src/bin/gs_loader/gs_loader.cpp +++ b/src/bin/gs_loader/gs_loader.cpp @@ -127,7 +127,7 @@ int main(int argc, char **argv) // get the exe path for get shell script char abs_path[PATH_SIZE] = {'\0'}; int cnt = readlink("/proc/self/exe", abs_path, PATH_SIZE); - if (cnt == -1 || cnt >= PATH_SIZE) { + if (cnt <= -1 || cnt >= PATH_SIZE) { std::cout << "ERROR: can not find gs_loader path" << std::endl; return 0; } diff --git a/src/bin/pg_basebackup/pg_basebackup.cpp b/src/bin/pg_basebackup/pg_basebackup.cpp index 04ffb2b28..05beb0e98 100644 --- a/src/bin/pg_basebackup/pg_basebackup.cpp +++ b/src/bin/pg_basebackup/pg_basebackup.cpp @@ -950,7 +950,7 @@ static void ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum) /* * All files are padded up to 512 bytes */ - if (current_len_left < 0 || current_len_left > INT_MAX - 511) { + if (current_len_left > INT_MAX - 511) { pg_log(stderr, _("%s: the file '%s' is too big or file size is invalid\n"), progname, copybuf); disconnect_and_exit(1); } diff --git a/src/bin/pg_ctl/backup.cpp b/src/bin/pg_ctl/backup.cpp index b4c65ff5c..a1b64c382 100755 --- a/src/bin/pg_ctl/backup.cpp +++ b/src/bin/pg_ctl/backup.cpp @@ -859,7 +859,7 @@ static bool ReceiveAndUnpackTarFile(PGconn* conn, PGresult* res, int rownum) /* * All files are padded up to 512 bytes */ - if (current_len_left < 0 || current_len_left > INT_MAX - 511) { + if (current_len_left > INT_MAX - 511) { pg_log(PG_WARNING, _("current_len_left is invalid\n")); DisconnectConnection(); FREE_AND_RESET(copybuf); diff --git a/src/bin/pg_dump/pg_backup_tar.cpp b/src/bin/pg_dump/pg_backup_tar.cpp index c9043e225..3d09641e1 100644 --- a/src/bin/pg_dump/pg_backup_tar.cpp +++ b/src/bin/pg_dump/pg_backup_tar.cpp @@ -555,7 +555,7 @@ static size_t tarRead(void* buf, size_t len, TAR_MEMBER* th) if (len > (size_t)(th->fileLen - th->pos)) len = th->fileLen - th->pos; - if (len <= 0) + if (len == 0) return 0; res = _tarReadRaw(th->AH, buf, len, th, NULL); diff --git a/src/bin/pg_probackup/catalog.cpp b/src/bin/pg_probackup/catalog.cpp index 2d1f50a82..5f065a979 100644 --- a/src/bin/pg_probackup/catalog.cpp +++ b/src/bin/pg_probackup/catalog.cpp @@ -1214,7 +1214,7 @@ void get_anchor_backup(timelineInfo *tlinfo, int *count, InstanceConfig *instanc /* sanity */ if (XLogRecPtrIsInvalid(backup->start_lsn) || - backup->tli <= 0) + backup->tli == 0) continue; /* skip invalid backups */ @@ -1393,7 +1393,7 @@ void anchor_lsn_keep_segments_timelines(InstanceConfig *instance, parray *timeli /* sanity for closest_backup */ if (XLogRecPtrIsInvalid(tlinfo->closest_backup->start_lsn) || - tlinfo->closest_backup->tli <= 0) + tlinfo->closest_backup->tli == 0) continue; /* @@ -1476,7 +1476,7 @@ void anchor_lsn_keep_segments_timelines(InstanceConfig *instance, parray *timeli /* sanity */ if (XLogRecPtrIsInvalid(backup->start_lsn) || - backup->tli <= 0) + backup->tli == 0) continue; /* no point in clogging keep_segments by backups protected by anchor_lsn */ @@ -1592,7 +1592,7 @@ catalog_get_timelines(InstanceConfig *instance) timelineinfos = walk_files_collect_timelines(instance); /* determine which WAL segments must be kept because of wal retention */ - if (instance->wal_depth <= 0) + if (instance->wal_depth == 0) return timelineinfos; /* diff --git a/src/bin/pg_probackup/delete.cpp b/src/bin/pg_probackup/delete.cpp index bedce29f4..0dc96d58a 100644 --- a/src/bin/pg_probackup/delete.cpp +++ b/src/bin/pg_probackup/delete.cpp @@ -701,11 +701,11 @@ do_retention_wal(bool dry_run) * at least one backup and no file should be removed. * Unless wal-depth is enabled. */ - if ((tlinfo->closest_backup) && instance_config.wal_depth <= 0) + if ((tlinfo->closest_backup) && instance_config.wal_depth == 0) continue; /* WAL retention keeps this timeline from purge */ - if (instance_config.wal_depth >= 0 && tlinfo->anchor_tli > 0 && + if (tlinfo->anchor_tli > 0 && tlinfo->anchor_tli != tlinfo->tli) continue; @@ -720,7 +720,7 @@ do_retention_wal(bool dry_run) */ if (tlinfo->oldest_backup) { - if (instance_config.wal_depth >= 0 && !(XLogRecPtrIsInvalid(tlinfo->anchor_lsn))) + if (!(XLogRecPtrIsInvalid(tlinfo->anchor_lsn))) { delete_walfiles_in_tli(tlinfo->anchor_lsn, tlinfo, instance_config.xlog_seg_size, dry_run); @@ -733,7 +733,7 @@ do_retention_wal(bool dry_run) } else { - if (instance_config.wal_depth >= 0 && !(XLogRecPtrIsInvalid(tlinfo->anchor_lsn))) + if (!(XLogRecPtrIsInvalid(tlinfo->anchor_lsn))) delete_walfiles_in_tli(tlinfo->anchor_lsn, tlinfo, instance_config.xlog_seg_size, dry_run); else @@ -996,7 +996,7 @@ delete_walfiles_in_tli(XLogRecPtr keep_lsn, timelineInfo *tlinfo, join_path_components(wal_fullpath, instance_config.arclog_path, wal_file->file.name); /* save segment from purging */ - if (instance_config.wal_depth >= 0 && wal_file->keep) + if (wal_file->keep) { elog(VERBOSE, "Retain WAL segment \"%s\"", wal_fullpath); continue; @@ -1146,12 +1146,9 @@ do_delete_status(InstanceConfig *instance_config, const char *status) } /* Inform about data size to free */ - if (size_to_delete >= 0) - { - pretty_size(size_to_delete, size_to_delete_pretty, lengthof(size_to_delete_pretty)); - elog(INFO, "Resident data size to free by delete of %i backups: %s", - n_deleted, size_to_delete_pretty); - } + pretty_size(size_to_delete, size_to_delete_pretty, lengthof(size_to_delete_pretty)); + elog(INFO, "Resident data size to free by delete of %i backups: %s", + n_deleted, size_to_delete_pretty); /* delete selected backups */ if (!dry_run && n_deleted > 0) diff --git a/src/bin/pg_probackup/logger.cpp b/src/bin/pg_probackup/logger.cpp index f1bd7d30f..b1de26ca8 100644 --- a/src/bin/pg_probackup/logger.cpp +++ b/src/bin/pg_probackup/logger.cpp @@ -540,7 +540,7 @@ logfile_getname(const char *format, time_t timestamp) #ifdef WIN32 if (pg_strftime(filename + len, MAXPGPATH - len, format, tm) <= 0) #else - if (strftime(filename + len, MAXPGPATH - len, format, tm) <= 0) + if (strftime(filename + len, MAXPGPATH - len, format, tm) == 0) #endif elog_stderr(ERROR, "strftime(%s) failed: %s", format, strerror(errno)); From cda13436de7e9b49fc82c9b8414fd42acc721320 Mon Sep 17 00:00:00 2001 From: ytwx1993 Date: Fri, 21 Jul 2023 10:49:18 +0800 Subject: [PATCH 041/304] =?UTF-8?q?common=E5=92=8Ccontrib=E8=B7=AF?= =?UTF-8?q?=E5=BE=84=E4=B8=8B=E5=AD=98=E5=9C=A8=E6=A0=BC=E5=BC=8F=E5=8C=96?= =?UTF-8?q?=E4=BD=BF=E7=94=A8=E9=94=99=E8=AF=AF=E5=9C=BA=E6=99=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- contrib/pg_check_clog/pg_check_clog.cpp | 4 ++-- contrib/pg_check_replslot/pg_check_replslot.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/contrib/pg_check_clog/pg_check_clog.cpp b/contrib/pg_check_clog/pg_check_clog.cpp index 26ab71087..a320dc998 100644 --- a/contrib/pg_check_clog/pg_check_clog.cpp +++ b/contrib/pg_check_clog/pg_check_clog.cpp @@ -103,7 +103,7 @@ static void parse_clog_file(const int segnum) for (bit_index = 0; bit_index < 4; bit_index++) { bshift = TransactionIdToBIndex(xid) * CLOG_BITS_PER_XACT; status = (*byteptr >> bshift) & CLOG_XACT_BITMASK; - fprintf(stdout, "xid %u, status: %s\n", xid, xid_status_name[status]); + fprintf(stdout, "xid %lu, status: %s\n", xid, xid_status_name[status]); xid++; } @@ -159,7 +159,7 @@ static void parse_single_xid(const TransactionId xid) byteptr = buffer + byteno; status = (*byteptr >> bshift) & CLOG_XACT_BITMASK; - fprintf(stdout, "xid %u, status: %s\n", xid, xid_status_name[status]); + fprintf(stdout, "xid %lu, status: %s\n", xid, xid_status_name[status]); if (close(fd)) { fprintf(stderr, "file close error !\n"); diff --git a/contrib/pg_check_replslot/pg_check_replslot.cpp b/contrib/pg_check_replslot/pg_check_replslot.cpp index 472d06c91..15c16bee0 100644 --- a/contrib/pg_check_replslot/pg_check_replslot.cpp +++ b/contrib/pg_check_replslot/pg_check_replslot.cpp @@ -199,7 +199,7 @@ static void show_slot(char* buffer, size_t size) char lsn[64] = {0}; if (size != sizeof(ReplicationSlotOnDisk)) { - fprintf(stderr, _("unexpected slot file size %d, expected %d\n"), (int)size, sizeof(ReplicationSlotOnDisk)); + fprintf(stderr, _("unexpected slot file size %d, expected %lu\n"), (int)size, sizeof(ReplicationSlotOnDisk)); exit(1); } @@ -237,7 +237,7 @@ static void show_slot(char* buffer, size_t size) snprintf(lsn, 63, "%X/%X", (uint32)(slot.slotdata.restart_lsn >> 32), (uint32)slot.slotdata.restart_lsn); fprintf(stderr, - _("%-24s| %-12s| %-8u| %-8u| %-12s| %-12s\n"), + _("%-24s| %-12s| %-8u| %-8lu| %-12s| %-12s\n"), NameStr(slot.slotdata.name), (slot.slotdata.database == InvalidOid) ? "physical" : "logical", slot.slotdata.database, From 0c1f1442ee734ccac411e5202853d0ad5999565e Mon Sep 17 00:00:00 2001 From: ytwx1993 Date: Fri, 21 Jul 2023 11:00:30 +0800 Subject: [PATCH 042/304] =?UTF-8?q?bool=E7=B1=BB=E5=9E=8B=E4=B8=8D?= =?UTF-8?q?=E8=83=BD=E7=94=A8null=E8=B5=8B=E5=80=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/psql/mainloop.cpp | 2 +- src/common/backend/utils/adt/ruleutils.cpp | 2 +- src/common/backend/utils/misc/guc.cpp | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/bin/psql/mainloop.cpp b/src/bin/psql/mainloop.cpp index 436af81ef..73f59c468 100644 --- a/src/bin/psql/mainloop.cpp +++ b/src/bin/psql/mainloop.cpp @@ -24,7 +24,7 @@ bool isSlashEnd(const char* strLine) { if (strLine == NULL) { - return NULL; + return false; } const char* pStr = strLine; while ('\0' != *pStr) { diff --git a/src/common/backend/utils/adt/ruleutils.cpp b/src/common/backend/utils/adt/ruleutils.cpp index 425e9f061..3409f3513 100644 --- a/src/common/backend/utils/adt/ruleutils.cpp +++ b/src/common/backend/utils/adt/ruleutils.cpp @@ -10591,7 +10591,7 @@ static void get_agg_expr(Aggref* aggref, deparse_context* context) aggform = (Form_pg_aggregate)GETSTRUCT(aggTuple); if (OidIsValid(aggform->aggfinalfn)) { - appendStringInfo(buf, "%s(", generate_function_name(aggform->aggfinalfn, 0, NULL, NULL, NULL, NULL)); + appendStringInfo(buf, "%s(", generate_function_name(aggform->aggfinalfn, 0, NULL, NULL, false, NULL)); added_finalfn = true; } ReleaseSysCache(aggTuple); diff --git a/src/common/backend/utils/misc/guc.cpp b/src/common/backend/utils/misc/guc.cpp index 024604f0c..9e8999e99 100755 --- a/src/common/backend/utils/misc/guc.cpp +++ b/src/common/backend/utils/misc/guc.cpp @@ -1878,7 +1878,7 @@ static void InitConfigureNamesBool() NULL, NULL, NULL, - NULL + false }, {{"enable_iud_fusion", PGC_USERSET, @@ -1891,7 +1891,7 @@ static void InitConfigureNamesBool() NULL, NULL, NULL, - NULL + false }, #endif {{"enable_expr_fusion", @@ -1905,7 +1905,7 @@ static void InitConfigureNamesBool() NULL, NULL, NULL, - NULL + false }, {{"ts_adaptive_threads", PGC_SIGHUP, From 19911b4dfa2eb05f296165d6da957bab94aaaa25 Mon Sep 17 00:00:00 2001 From: pulsar Date: Fri, 21 Jul 2023 17:11:10 +0800 Subject: [PATCH 043/304] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=9C=9F=E6=9C=9B?= =?UTF-8?q?=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/test/regress/expected/plpgsql_cursor_rowtype.out | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/regress/expected/plpgsql_cursor_rowtype.out b/src/test/regress/expected/plpgsql_cursor_rowtype.out index 1981e9116..8b7665a1f 100644 --- a/src/test/regress/expected/plpgsql_cursor_rowtype.out +++ b/src/test/regress/expected/plpgsql_cursor_rowtype.out @@ -988,7 +988,7 @@ NOTICE: drop cascades to function plpgsql_cursor_rowtype.p1() drop package pckg_test2; NOTICE: drop cascades to function plpgsql_cursor_rowtype.p1() drop schema plpgsql_cursor_rowtype cascade; -NOTICE: drop cascades to 13 other objects +NOTICE: drop cascades to 14 other objects DETAIL: drop cascades to table emp drop cascades to function pro_cursor_no_args_1() drop cascades to function pro_cursor_no_args_2() @@ -1002,5 +1002,6 @@ drop cascades to function test_forloop_001() drop cascades to function pro_close_cursor1() drop cascades to function pro_close_cursor2() drop cascades to function check_compile_1() +drop cascades to type foo drop schema schema1 cascade; NOTICE: drop cascades to table schema1.t11 From c8ce7f3a6a92ba2699298840b5ada822bfc22009 Mon Sep 17 00:00:00 2001 From: congzhou2603 Date: Thu, 20 Jul 2023 16:45:24 +0800 Subject: [PATCH 044/304] =?UTF-8?q?=E3=80=90bugfix=E3=80=91=20=E4=BF=AE?= =?UTF-8?q?=E6=94=B9=E5=88=86=E5=8C=BA=E8=A1=A8=E5=8E=8B=E7=BC=A9=E5=B1=9E?= =?UTF-8?q?=E6=80=A7=E6=97=B6=E6=9C=89=E5=8F=AF=E8=83=BD=E5=86=85=E5=AD=98?= =?UTF-8?q?=E6=BA=A2=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/optimizer/commands/cluster.cpp | 2 +- .../regress/expected/row_compression/alter_compress_params.out | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/gausskernel/optimizer/commands/cluster.cpp b/src/gausskernel/optimizer/commands/cluster.cpp index 94c258945..d4d62aa08 100755 --- a/src/gausskernel/optimizer/commands/cluster.cpp +++ b/src/gausskernel/optimizer/commands/cluster.cpp @@ -3219,7 +3219,7 @@ static void swapPartitionfiles( /* set the relOptions of old rel to the ones after compressed options */ UpdatePartitionRelOptionsSystemCache(tab->newOptions, &reltup1, relRelation1, (void**)&relform1); /* set the relOptions of new rel to the ones before alter compressed options */ - UpdatePartitionRelOptionsSystemCache(tab->oldOptions, &reltup2, relRelation2, (void**)&relform2); + UpdatePgClassRelOptionsSystemCache(tab->oldOptions, &reltup2, relRelation2, (void**)&relform2); } /* swap size statistics too, since new rel has freshly-updated stats */ diff --git a/src/test/regress/expected/row_compression/alter_compress_params.out b/src/test/regress/expected/row_compression/alter_compress_params.out index e0f4a353e..260e565c1 100644 --- a/src/test/regress/expected/row_compression/alter_compress_params.out +++ b/src/test/regress/expected/row_compression/alter_compress_params.out @@ -793,7 +793,7 @@ SELECT nchunks, chunknos FROM pg_catalog.compress_address_details('alter_compres 1 | {1} (1 row) -SELECT count(*) FROM compress_statistic_info(compress_func_findpath('alter_compress_params_schema.all_options_table_compresstype_1_cpc_1'), 1); +SELECT count(*) FROM compress_ratio_info(compress_func_findpath('alter_compress_params_schema.all_options_table_compresstype_1_cpc_1')); count ------- 1 From f0e67b816ea29affb72e3b2dba7146a695342fbe Mon Sep 17 00:00:00 2001 From: "arcoalien@qq.com" Date: Wed, 19 Jul 2023 20:55:28 +0800 Subject: [PATCH 045/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E8=B5=84=E6=BA=90?= =?UTF-8?q?=E6=B1=A0=E5=8C=96checkpoint=E5=88=9B=E5=BB=BA=E5=8F=91?= =?UTF-8?q?=E5=B8=83=E8=AE=A2=E9=98=85=E5=A4=8D=E5=88=B6=E6=A7=BD=E5=BC=82?= =?UTF-8?q?=E5=B8=B8=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/storage/replication/logical/origin.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/gausskernel/storage/replication/logical/origin.cpp b/src/gausskernel/storage/replication/logical/origin.cpp index 826a69b14..4bed2d34f 100644 --- a/src/gausskernel/storage/replication/logical/origin.cpp +++ b/src/gausskernel/storage/replication/logical/origin.cpp @@ -438,7 +438,7 @@ Size ReplicationOriginShmemSize(void) * we keep the replay state of *remote* transactions. But for now it seems * sufficient to reuse it, lest we introduce a separate guc. */ - if (g_instance.attr.attr_storage.max_replication_slots == 0) + if (g_instance.attr.attr_storage.max_replication_slots == 0 || ENABLE_DMS) return size; size = add_size(size, offsetof(ReplicationStateShmStruct, states)); @@ -520,7 +520,7 @@ void CheckPointReplicationOrigin(void) pg_crc32c crc; struct stat st; - if (g_instance.attr.attr_storage.max_replication_slots == 0) { + if (g_instance.attr.attr_storage.max_replication_slots == 0 || ENABLE_DMS) { return; } @@ -645,7 +645,7 @@ void StartupReplicationOrigin(void) pg_crc32c file_crc; pg_crc32c crc; - if (g_instance.attr.attr_storage.max_replication_slots == 0) + if (g_instance.attr.attr_storage.max_replication_slots == 0 || ENABLE_DMS) return; INIT_CRC32C(crc); From 584e63c3822ef40af28335acbd6cf3d113b23970 Mon Sep 17 00:00:00 2001 From: chendong76 <1209756284@qq.com> Date: Mon, 24 Jul 2023 16:10:20 +0800 Subject: [PATCH 046/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dtcp=5Fuser=5Ftimeout?= =?UTF-8?q?=E5=8F=82=E6=95=B0=E5=A4=8D=E5=88=B6=E9=94=99=E8=AF=AF=E7=9A=84?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/utils/misc/guc.cpp | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/common/backend/utils/misc/guc.cpp b/src/common/backend/utils/misc/guc.cpp index 2bd38761e..58629461c 100755 --- a/src/common/backend/utils/misc/guc.cpp +++ b/src/common/backend/utils/misc/guc.cpp @@ -498,9 +498,11 @@ static void pg_timezone_abbrev_initialize(void); static void assign_tcp_keepalives_idle(int newval, void *extra); static void assign_tcp_keepalives_interval(int newval, void *extra); static void assign_tcp_keepalives_count(int newval, void *extra); +static void assign_tcp_user_timeout(int newval, void *extra); static const char* show_tcp_keepalives_idle(void); static const char* show_tcp_keepalives_interval(void); static const char* show_tcp_keepalives_count(void); +static const char* show_tcp_user_timeout(void); static bool check_effective_io_concurrency(int* newval, void** extra, GucSource source); static void assign_effective_io_concurrency(int newval, void* extra); static void assign_pgstat_temp_directory(const char* newval, void* extra); @@ -2542,8 +2544,8 @@ static void InitConfigureNamesInt() 0, 3600000, NULL, - assign_tcp_keepalives_count, - show_tcp_keepalives_count}, + assign_tcp_user_timeout, + show_tcp_user_timeout}, {{"gin_fuzzy_search_limit", PGC_USERSET, NODE_ALL, @@ -11958,6 +11960,21 @@ static const char* show_tcp_keepalives_count(void) return nbuf; } +static void assign_tcp_user_timeout(int newval, void *extra) +{ + (void) pq_settcpusertimeout(newval, u_sess->proc_cxt.MyProcPort); +} + +static const char* show_tcp_user_timeout(void) +{ + const int maxBufLen = 16; + static char nbuf[maxBufLen]; + + errno_t rc = snprintf_s(nbuf, maxBufLen, maxBufLen - 1, "%d", pq_gettcpusertimeout(u_sess->proc_cxt.MyProcPort)); + securec_check_ss(rc, "\0", "\0"); + return nbuf; +} + static bool check_effective_io_concurrency(int* newval, void** extra, GucSource source) { #ifdef USE_PREFETCH From 4eae2c171a65de0c9d7cf90ba95d3a0270042687 Mon Sep 17 00:00:00 2001 From: "arcoalien@qq.com" Date: Mon, 24 Jul 2023 16:43:05 +0800 Subject: [PATCH 047/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=A4=9A=E5=86=99?= =?UTF-8?q?=E5=BC=80=E5=90=AF=E5=90=8Epbe=E6=A8=A1=E5=BC=8F=E4=B8=AD?= =?UTF-8?q?=E4=BA=8B=E5=8A=A1=E5=8C=85=E5=90=AB=E6=9C=89DDL,=E5=A6=82?= =?UTF-8?q?=E6=9E=9C=E4=B8=80=E7=9B=B4=E7=AD=89=E5=88=B0ddl=E8=AF=AD?= =?UTF-8?q?=E5=8F=A5=E7=BB=8F=E8=BF=87PBE=E5=90=8E=E6=89=8D=E5=8F=91S?= =?UTF-8?q?=EF=BC=8C=E6=B2=A1=E6=9C=89=E6=8B=A6=E6=88=AA=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../storage/access/transam/xact.cpp | 2 +- .../storage/access/transam/xlog.cpp | 3 +- .../storage/replication/libpqsw.cpp | 51 +++++++++++-------- src/include/replication/libpqsw.h | 2 +- 4 files changed, 33 insertions(+), 25 deletions(-) diff --git a/src/gausskernel/storage/access/transam/xact.cpp b/src/gausskernel/storage/access/transam/xact.cpp index d4b47eaa0..8af3cbff8 100755 --- a/src/gausskernel/storage/access/transam/xact.cpp +++ b/src/gausskernel/storage/access/transam/xact.cpp @@ -4039,7 +4039,7 @@ static void AbortTransaction(bool PerfectRollback, bool STP_rollback) #endif if (SS_STANDBY_MODE_WITH_REMOTE_EXECUTE && !libpqsw_is_end()) { - libpqsw_disconnect(); + libpqsw_disconnect(true); } s->savepointList = NULL; diff --git a/src/gausskernel/storage/access/transam/xlog.cpp b/src/gausskernel/storage/access/transam/xlog.cpp index 4cd9ef70d..015d48803 100755 --- a/src/gausskernel/storage/access/transam/xlog.cpp +++ b/src/gausskernel/storage/access/transam/xlog.cpp @@ -707,8 +707,6 @@ static XLogRecPtr XLogInsertRecordGroup(XLogRecData *rdata, XLogRecPtr fpw_lsn) extra_waits++; } - Assert(pg_atomic_read_u32(&proc->xlogGroupNext) == INVALID_PGPROCNO); - #ifdef ENABLE_MULTIPLE_NODES /* Fix semaphore count for any absorbed wakeups */ while (extra_waits-- > 0) { @@ -722,6 +720,7 @@ static XLogRecPtr XLogInsertRecordGroup(XLogRecData *rdata, XLogRecPtr fpw_lsn) * in case of memory reordering in relaxed memory model like ARM. */ pg_memory_barrier(); + Assert(pg_atomic_read_u32(&proc->xlogGroupNext) == INVALID_PGPROCNO); return proc->xlogGroupReturntRecPtr; } diff --git a/src/gausskernel/storage/replication/libpqsw.cpp b/src/gausskernel/storage/replication/libpqsw.cpp index 697a5d136..3cede836f 100644 --- a/src/gausskernel/storage/replication/libpqsw.cpp +++ b/src/gausskernel/storage/replication/libpqsw.cpp @@ -55,7 +55,7 @@ int internal_putbytes(const char* s, size_t len); int pq_flush(void); PGresult* libpqsw_get_result(PGconn* conn, libpqsw_transfer_standby_func transfer_func); bool libpqsw_connect(char* conninfo, const char *dbName, const char* userName); -void libpqsw_disconnect(void); +void libpqsw_disconnect(bool clear_queue); bool libpqsw_send_pbe(const char* buffer, size_t buffer_size); bool libpqsw_begin_command(const char* commandTag); bool libpqsw_end_command(const char* commandTag); @@ -276,7 +276,7 @@ void libpqsw_create_conn() { uint32 ss_standby_state = get_redirect_manager()->ss_standby_state; RedirectState temp_state = get_redirect_manager()->state; - libpqsw_disconnect(); + libpqsw_disconnect(false); get_redirect_manager()->ss_standby_state = ss_standby_state; get_redirect_manager()->state = temp_state; if (SS_STANDBY_MODE) { @@ -301,7 +301,7 @@ static bool libpqsw_remote_excute_sql(int retry, const char *sql, uint32 size, c if (get_sw_cxt()->streamConn == NULL) { libpqsw_create_conn(); } else if(PQstatus(get_sw_cxt()->streamConn) != CONNECTION_OK) { - libpqsw_disconnect(); + libpqsw_disconnect(true); ereport(ERROR, (errcode(ERRCODE_CONNECTION_RESET_BY_PEER), errmsg("connection already bad!%s", @@ -318,7 +318,7 @@ static bool libpqsw_remote_excute_sql(int retry, const char *sql, uint32 size, c } else { // send failed, so we need retry! uint32 ss_standby_state = get_redirect_manager()->ss_standby_state; - libpqsw_disconnect(); + libpqsw_disconnect(false); get_redirect_manager()->ss_standby_state = ss_standby_state; if (retry > 0) { return libpqsw_remote_excute_sql(retry - 1, sql, size, commandTag, waitResult, transfer); @@ -361,14 +361,14 @@ bool libpqsw_can_seek_next_session() void libpqsw_cleanup(int code, Datum arg) { - if (u_sess == NULL) { + if (u_sess == NULL || !g_instance.attr.attr_sql.enableRemoteExcute) { return; } ereport(LIBPQSW_DEFAULT_LOG_LEVEL, (errmsg("libpqsw(%ld): cleanup called!", get_sw_cxt()->redirect_manager == NULL ? -1 : ((int64)(get_sw_cxt()->redirect_manager))))); if (get_sw_cxt()->streamConn != NULL) { - libpqsw_disconnect(); + libpqsw_disconnect(true); } if (get_sw_cxt()->redirect_manager != NULL) { DELETE_EX_TYPE(get_sw_cxt()->redirect_manager, RedirectManager); @@ -396,6 +396,15 @@ static bool libpqsw_before_redirect(const char* commandTag, List* query_list, co } bool need_redirect = false; redirect_manager->state.have_savepoint= false; + if (libpqsw_remote_in_transaction() || libpqsw_get_transaction()) { + if (!SS_STANDBY_MODE && set_command_type_by_commandTag(commandTag) == CMD_DDL && + !libpqsw_fetch_command(commandTag)) { + libpqsw_disconnect(true); + ereport(ERROR, (errmsg("The multi-write feature doesn't support DDL within transaction!"))); + } + libpqsw_check_savepoint(query_list, &(redirect_manager->state.have_savepoint)); + } + if (!libpqsw_enable_autocommit()) { if (strcmp(commandTag, "SET") == 0) { libpqsw_set_set_command(true); @@ -404,12 +413,6 @@ static bool libpqsw_before_redirect(const char* commandTag, List* query_list, co libpqsw_set_transaction(true); need_redirect = true; } else if (libpqsw_begin_command(commandTag) || libpqsw_remote_in_transaction()) { - if (!SS_STANDBY_MODE && set_command_type_by_commandTag(commandTag) == CMD_DDL && - !libpqsw_fetch_command(commandTag)) { - libpqsw_disconnect(); - ereport(ERROR, (errmsg("The multi-write feature doesn't support DDL within transaction!"))); - } - libpqsw_check_savepoint(query_list, &(redirect_manager->state.have_savepoint)); libpqsw_set_transaction(true); need_redirect = true; } else if (libpqsw_redirect()) { @@ -856,7 +859,7 @@ static bool libpqsw_need_localexec_forSimpleQuery(const char *commandTag, List * get_redirect_manager()->ss_standby_state |= SS_STANDBY_REQ_WRITE_REDIRECT; return ret; } else { - libpqsw_disconnect(); + libpqsw_disconnect(true); ereport(ERROR, (errmsg("The multi-write feature doesn't support DDL within transaction!"))); } } @@ -967,7 +970,7 @@ bool libpqsw_process_message(int qtype, StringInfo msg) // exit msg if (qtype == 'X' || qtype == -1) { libpqsw_receive(true); - libpqsw_disconnect(); + libpqsw_disconnect(true); return false; } // process U B E msg @@ -1058,7 +1061,7 @@ bool libpqsw_process_query_message(const char* commandTag, List* query_list, con } if (get_sw_cxt()->streamConn->xactStatus == PQTRANS_INERROR) { - libpqsw_disconnect(); + libpqsw_disconnect(true); ereport(ERROR, (errmsg("The primary node report error when last request was transferred to it!"))); } @@ -1229,12 +1232,18 @@ bool libpqsw_connect(char* conninfo, const char *dbName, const char* userName) /* * Disconnect connection to primary, if any. */ -void libpqsw_disconnect(void) +void libpqsw_disconnect(bool clear_queue) { + RedirectManager* redirect_manager = (RedirectManager*)get_sw_cxt()->redirect_manager; ereport(LIBPQSW_DEFAULT_LOG_LEVEL, (errmsg("libpqsw(%ld): libpqsw_disconnect called, conn is null:%s", - get_sw_cxt()->redirect_manager == NULL ? -1 : ((int64)(get_sw_cxt()->redirect_manager)), + redirect_manager == NULL ? -1 : ((int64)(redirect_manager)), get_sw_cxt()->streamConn == NULL ? "true" : "false"))); + RedirectMessageManager* message_manager = &(redirect_manager->messages_manager); + if (clear_queue && !(message_manager->message_empty())) { + message_manager->reset(); + } + if (get_sw_cxt()->streamConn != NULL) { if (get_sw_cxt()->conn_trace_file != NULL) { PQuntrace(get_sw_cxt()->streamConn); @@ -1245,8 +1254,8 @@ void libpqsw_disconnect(void) get_sw_cxt()->streamConn = NULL; } - if (get_sw_cxt()->redirect_manager != NULL) { - get_redirect_manager()->init(); + if (redirect_manager != NULL) { + redirect_manager->init(); } } @@ -1379,7 +1388,7 @@ bool libpqsw_send_pbe(const char* buffer, size_t buffer_size) struct pg_conn* conn = get_sw_cxt()->streamConn; bool result = true; if (!libpqsw_before_send(conn)) { - libpqsw_disconnect(); + libpqsw_disconnect(true); result = false; ereport(ERROR, (errcode(ERRCODE_CONNECTION_FAILURE), @@ -1387,7 +1396,7 @@ bool libpqsw_send_pbe(const char* buffer, size_t buffer_size) } conn->outMsgEnd = conn->outMsgStart = conn->outCount; if (pqPutnchar(buffer, buffer_size, conn) < 0) { - libpqsw_disconnect(); + libpqsw_disconnect(true); result = false; ereport(ERROR, (errcode(ERRCODE_INVALID_STATUS), diff --git a/src/include/replication/libpqsw.h b/src/include/replication/libpqsw.h index b2b9bcb69..aa837ac84 100644 --- a/src/include/replication/libpqsw.h +++ b/src/include/replication/libpqsw.h @@ -84,7 +84,7 @@ bool libpqsw_is_end(); bool libpqsw_only_localrun(); void libpqsw_create_conn(); void libpqsw_trace_q_msg(const char* commandTag, const char* queryString); -void libpqsw_disconnect(void); +void libpqsw_disconnect(bool clear_queue); void libpqsw_check_ddl_on_primary(const char* commandTag); #ifdef _cplusplus From ff065ec13c59f8921a39bf42174cc7a17b26d332 Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Mon, 24 Jul 2023 20:27:33 +0800 Subject: [PATCH 048/304] =?UTF-8?q?=E8=A7=A3=E5=86=B3gs=5Fctl=E5=86=85?= =?UTF-8?q?=E5=AD=98=E6=B3=84=E6=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/pg_ctl/pg_ctl.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/bin/pg_ctl/pg_ctl.cpp b/src/bin/pg_ctl/pg_ctl.cpp index 42ecb29f2..e2c8f9a5b 100755 --- a/src/bin/pg_ctl/pg_ctl.cpp +++ b/src/bin/pg_ctl/pg_ctl.cpp @@ -7331,6 +7331,8 @@ static void free_ctl() FREE_AND_RESET(register_password); FREE_AND_RESET(pgha_str); FREE_AND_RESET(pgha_opt); + FREE_AND_RESET(ss_instance_config.dss.vgname); + FREE_AND_RESET(ss_instance_config.dss.vgdata); } static int get_instance_id(void) @@ -7457,7 +7459,9 @@ bool ss_read_config(void) (void)find_guc_optval((const char**)optlines, "ss_enable_dss", enable_dss); /* this is not enable_dss, wo do not need to do anythiny else */ - if(strncmp(enable_dss, "on", sizeof("on")) != 0) { + if(strcmp(enable_dss, "on") != 0) { + freefile(optlines); + optlines = NULL; return false; } From 33bbeb4f25c20c615887f9eecc2f547dc3e69d02 Mon Sep 17 00:00:00 2001 From: duzhuolin Date: Tue, 25 Jul 2023 09:35:13 +0800 Subject: [PATCH 049/304] =?UTF-8?q?WLM=E8=BF=9E=E6=8E=A5=E8=B6=85=E6=97=B6?= =?UTF-8?q?=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/cbb/workload/statctl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gausskernel/cbb/workload/statctl.cpp b/src/gausskernel/cbb/workload/statctl.cpp index 798cbe9e8..0596bd7bf 100644 --- a/src/gausskernel/cbb/workload/statctl.cpp +++ b/src/gausskernel/cbb/workload/statctl.cpp @@ -1866,7 +1866,7 @@ void WLMReadjustUserSpaceByQuery(const char* username, List* database_name_list) errno_t rc = snprintf_s(conninfo, sizeof(conninfo), sizeof(conninfo) - 1, - "dbname=%s port=%d application_name='statctl'", + "dbname=%s port=%d application_name='statctl' connect_timeout=5", lfirst(cell), g_instance.attr.attr_network.PostPortNumber); securec_check_ss(rc, "\0", "\0"); From 56f19ce8bbd03b19e7034dd48181851dd37cdf3c Mon Sep 17 00:00:00 2001 From: bowenliu Date: Wed, 7 Jun 2023 20:42:46 +0800 Subject: [PATCH 050/304] SS standby nodes Consistent Read logic enhancement ss visibility prefview dfx; upgrade and rollback sqls. fix bugs - ss visibility inconsistency bug - add debug version validation --- src/common/backend/catalog/builtin_funcs.ini | 4 + src/common/backend/utils/adt/pgstatfuncs.cpp | 60 ++++ src/common/backend/utils/error/be_module.cpp | 1 + .../backend/utils/misc/guc/guc_storage.cpp | 33 ++ src/gausskernel/ddes/adapter/Makefile | 2 +- src/gausskernel/ddes/adapter/ss_init.cpp | 11 + .../ddes/adapter/ss_transaction.cpp | 112 +++++- src/gausskernel/ddes/adapter/ss_txnstatus.cpp | 334 ++++++++++++++++++ .../process/threadpool/knl_thread.cpp | 2 + src/gausskernel/storage/ipc/ipci.cpp | 3 + src/gausskernel/storage/lmgr/lwlock.cpp | 10 +- .../rollback-post_catalog_maindb_92_906.sql | 1 + .../rollback-post_catalog_otherdb_92_906.sql | 1 + .../upgrade-post_catalog_maindb_92_906.sql | 15 + .../upgrade-post_catalog_otherdb_92_906.sql | 15 + src/include/ddes/dms/ss_common_attr.h | 12 + src/include/ddes/dms/ss_txnstatus.h | 66 ++++ .../knl/knl_guc/knl_instance_attr_storage.h | 1 + src/include/knl/knl_instance.h | 1 + src/include/knl/knl_thread.h | 3 + src/include/miscadmin.h | 1 + src/include/storage/lock/lwlock.h | 9 +- src/include/utils/be_module.h | 1 + src/test/regress/expected/opr_sanity.out | 1 + .../expected/single_node_opr_sanity.out | 1 + .../regress/output/recovery_2pc_tools.source | 1 + 26 files changed, 696 insertions(+), 5 deletions(-) create mode 100644 src/gausskernel/ddes/adapter/ss_txnstatus.cpp create mode 100644 src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback-post_catalog_maindb_92_906.sql create mode 100644 src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback-post_catalog_otherdb_92_906.sql create mode 100644 src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_906.sql create mode 100644 src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_906.sql create mode 100644 src/include/ddes/dms/ss_txnstatus.h diff --git a/src/common/backend/catalog/builtin_funcs.ini b/src/common/backend/catalog/builtin_funcs.ini index ef8d3f4de..2ea8be3fa 100755 --- a/src/common/backend/catalog/builtin_funcs.ini +++ b/src/common/backend/catalog/builtin_funcs.ini @@ -10987,6 +10987,10 @@ AddFuncGroup( "ss_buffer_ctrl", 1, AddBuiltinFunc(_0(4214), _1("ss_buffer_ctrl"), _2(0), _3(false), _4(true), _5(ss_buffer_ctrl), _6(2249), _7(PG_CATALOG_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(100), _12(0), _13(0), _14(false), _15(false), _16(false), _17(false), _18('s'), _19(0), _20(0), _21(13, 23, 5545, 5545, 5545, 5545, 5545, 23, 26, 26, 26, 20, 5545, 26), _22(13, 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o'), _23(13, "bufferid", "is_remote_dirty", "lock_mode", "is_edp", "force_request", "need_flush", "buf_id", "state", "pblk_relno", "pblk_blkno", "pblk_lsn", "seg_fileno", "seg_blockno"), _24(NULL), _25("ss_buffer_ctrl"), _26(NULL), _27(NULL), _28(NULL), _29(0), _30(false), _31(NULL), _32(false), _33(NULL), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)) ), + AddFuncGroup( + "ss_txnstatus_cache_stat", 1, + AddBuiltinFunc(_0(8888), _1("ss_txnstatus_cache_stat"), _2(0), _3(true), _4(true), _5(ss_txnstatus_cache_stat), _6(2249), _7(PG_CATALOG_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(1000), _12(0), _13(0), _14(false), _15(false), _16(false), _17(false), _18('v'), _19(0), _20(0), _21(8, 20, 20, 20, 701, 701, 701, 20, 701), _22(8, 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o'), _23(8, "vcache_gets", "hcache_gets", "nio_gets", "avg_hcache_gettime_us", "avg_nio_gettime_us", "cache_hit_rate", "hcache_eviction", "avg_eviction_refcnt"), _24(NULL), _25("ss_txnstatus_cache_stat"), _26(NULL), _27(NULL), _28(NULL), _29(0), _30(false), _31(NULL), _32(false), _33(NULL), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)) + ), AddFuncGroup( "start_collect_workload", 1, AddBuiltinFunc(_0(7810), _1("start_collect_workload"), _2(2), _3(true), _4(false), _5(start_collect_workload), _6(16), _7(PG_SQLADVISOR_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(0), _12(0), _13(0), _14(false), _15(false), _16(false), _17(false), _18('v'), _19(2), _20(2, 23, 23), _21(NULL), _22(NULL), _23(NULL), _24("({CONST :consttype 23 :consttypmod -1 :constcollid 0 :constlen 4 :constbyval true :constisnull false :ismaxvalue false :location 27 :constvalue 4 [ 16 39 0 0 0 0 0 0 ] :cursor_data :row_count 0 :cur_dno -1 :is_open false :found false :not_found false :null_open false :null_fetch false} {CONST :consttype 23 :consttypmod -1 :constcollid 0 :constlen 4 :constbyval true :constisnull false :ismaxvalue false :location 27 :constvalue 4 [ 0 4 0 0 0 0 0 0 ] :cursor_data :row_count 0 :cur_dno -1 :is_open false :found false :not_found false :null_open false :null_fetch false})"), _25("start_collect_workload"), _26(NULL), _27(NULL), _28(NULL), _29(2, 0, 1), _30(false), _31(NULL), _32(false), _33(NULL), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)) diff --git a/src/common/backend/utils/adt/pgstatfuncs.cpp b/src/common/backend/utils/adt/pgstatfuncs.cpp index dcd9e61e2..01e1d7966 100644 --- a/src/common/backend/utils/adt/pgstatfuncs.cpp +++ b/src/common/backend/utils/adt/pgstatfuncs.cpp @@ -9264,6 +9264,66 @@ void insert_pv_session_memory(Tuplestorestate* tupStore, TupleDesc tupDesc, cons } +static inline double calcCacheHitRate() { + uint64 cache = g_instance.dms_cxt.SSDFxStats.txnstatus_varcache_gets + + g_instance.dms_cxt.SSDFxStats.txnstatus_hashcache_gets; + uint64 total = cache + g_instance.dms_cxt.SSDFxStats.txnstatus_network_io_gets; + double hitrate = total == 0 ? 0 : ((cache * 1.0) / total); + return hitrate; +} + +/* txnstatus_cache_stat view pg_tde_info */ +Datum ss_txnstatus_cache_stat(PG_FUNCTION_ARGS) +{ + int i = 0; + errno_t rc = 0; + + TupleDesc tupdesc = CreateTemplateTupleDesc(8, false); + TupleDescInitEntry(tupdesc, (AttrNumber)++i, "vcache_gets", INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)++i, "hcache_gets", INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)++i, "nio_gets", INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)++i, "avg_hcache_gettime_us", FLOAT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)++i, "avg_nio_gettime_us", FLOAT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)++i, "cache_hit_rate", FLOAT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)++i, "hcache_eviction", INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)++i, "avg_eviction_refcnt", FLOAT8OID, -1, 0); + BlessTupleDesc(tupdesc); + + Datum values[8]; + bool nulls[8] = {false}; + HeapTuple tuple = NULL; + + rc = memset_s(values, sizeof(values), 0, sizeof(values)); + securec_check(rc, "\0", "\0"); + rc = memset_s(nulls, sizeof(nulls), 0, sizeof(nulls)); + securec_check(rc, "\0", "\0"); + + i = 0; + double avgnio = g_instance.dms_cxt.SSDFxStats.txnstatus_network_io_gets == 0 ? 0 : + (g_instance.dms_cxt.SSDFxStats.txnstatus_total_niogets_time * 1.0 / + g_instance.dms_cxt.SSDFxStats.txnstatus_network_io_gets); + double avghash = g_instance.dms_cxt.SSDFxStats.txnstatus_hashcache_gets == 0 ? 0 : + (g_instance.dms_cxt.SSDFxStats.txnstatus_total_hcgets_time * 1.0 / + g_instance.dms_cxt.SSDFxStats.txnstatus_hashcache_gets); + double avgref = g_instance.dms_cxt.SSDFxStats.txnstatus_total_evictions == 0 ? 0 : + (g_instance.dms_cxt.SSDFxStats.txnstatus_total_eviction_refcnt * 1.0 / + g_instance.dms_cxt.SSDFxStats.txnstatus_total_evictions); + + values[i++] = Int64GetDatum(g_instance.dms_cxt.SSDFxStats.txnstatus_varcache_gets); + values[i++] = Int64GetDatum(g_instance.dms_cxt.SSDFxStats.txnstatus_hashcache_gets); + values[i++] = Int64GetDatum(g_instance.dms_cxt.SSDFxStats.txnstatus_network_io_gets); + values[i++] = Float8GetDatum(avghash); + values[i++] = Float8GetDatum(avgnio); + values[i++] = Float8GetDatum(calcCacheHitRate()); + values[i++] = Int64GetDatum(g_instance.dms_cxt.SSDFxStats.txnstatus_total_evictions); + values[i] = Float8GetDatum(avgref); + + tuple = heap_form_tuple(tupdesc, values, nulls); + + PG_RETURN_DATUM(HeapTupleGetDatum(tuple)); +} + + Datum pg_stat_get_file_stat(PG_FUNCTION_ARGS) { FuncCallContext* funcctx = NULL; diff --git a/src/common/backend/utils/error/be_module.cpp b/src/common/backend/utils/error/be_module.cpp index f74db8b61..6d2d0d7e4 100755 --- a/src/common/backend/utils/error/be_module.cpp +++ b/src/common/backend/utils/error/be_module.cpp @@ -136,6 +136,7 @@ const module_data module_map[] = {{MOD_ALL, "ALL"}, {MOD_GPI, "GPI"}, {MOD_PARTITION, "PARTITION"}, {MOD_SRF, "SRF"}, + {MOD_SS_TXNSTATUS, "SS_TXNSTATUS"}, /* add your module name above */ {MOD_MAX, "BACKEND"}}; diff --git a/src/common/backend/utils/misc/guc/guc_storage.cpp b/src/common/backend/utils/misc/guc/guc_storage.cpp index 196b32fb8..b98533c0a 100755 --- a/src/common/backend/utils/misc/guc/guc_storage.cpp +++ b/src/common/backend/utils/misc/guc/guc_storage.cpp @@ -220,6 +220,7 @@ static bool check_ss_enable_ondemand_recovery(bool* newval, void** extra, GucSou #ifdef USE_ASSERT_CHECKING static void assign_ss_enable_verify_page(bool newval, void *extra); #endif +static bool check_ss_txnstatus_cache_size(int* newval, void** extra, GucSource source); #ifndef ENABLE_MULTIPLE_NODES static void assign_dcf_election_timeout(int newval, void* extra); @@ -3667,6 +3668,20 @@ static void InitStorageConfigureNamesInt() NULL, NULL, NULL}, + {{"ss_txnstatus_cache_size", + PGC_POSTMASTER, + NODE_SINGLENODE, + SHARED_STORAGE_OPTIONS, + gettext_noop("Number of entries in txnstatus_cache"), + NULL, + GUC_SUPERUSER_ONLY}, + &g_instance.attr.attr_storage.dms_attr.txnstatus_cache_size, + 131072, + 0, + 524288, + check_ss_txnstatus_cache_size, + NULL, + NULL}, /* End-of-list marker */ {{NULL, (GucContext)0, @@ -6107,6 +6122,24 @@ static void assign_ss_enable_verify_page(bool newval, void *extra) } #endif +static bool check_ss_txnstatus_cache_size(int* newval, void** extra, GucSource source) +{ + if (*newval == 0) { + return true; + } + + const int minval = 8192; + if (*newval < minval) { + ereport(FATAL, (errmsg("ss_txnstatus_cache_size set as %d, should be >8192 or 0.", *newval))); + } + + if (*newval % NUM_TXNSTATUS_CACHE_PARTITIONS != 0) { + ereport(FATAL, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("ss_txnstatus_cache_size should be multiple of partition number 256."))); + } + return true; +} + #ifndef ENABLE_MULTIPLE_NODES static void assign_dcf_election_timeout(int newval, void* extra) diff --git a/src/gausskernel/ddes/adapter/Makefile b/src/gausskernel/ddes/adapter/Makefile index 785a286c6..121134277 100644 --- a/src/gausskernel/ddes/adapter/Makefile +++ b/src/gausskernel/ddes/adapter/Makefile @@ -22,7 +22,7 @@ ifneq "$(MAKECMDGOALS)" "clean" endif OBJS = ss_dms_bufmgr.o ss_dms_callback.o ss_dms_recovery.o ss_dms.o ss_init.o \ - ss_reform_common.o ss_switchover.o ss_transaction.o ss_aio.o + ss_reform_common.o ss_switchover.o ss_transaction.o ss_aio.o ss_txnstatus.o include $(top_srcdir)/src/gausskernel/common.mk diff --git a/src/gausskernel/ddes/adapter/ss_init.cpp b/src/gausskernel/ddes/adapter/ss_init.cpp index 9a286a31a..54b437919 100644 --- a/src/gausskernel/ddes/adapter/ss_init.cpp +++ b/src/gausskernel/ddes/adapter/ss_init.cpp @@ -387,6 +387,16 @@ static void setDMSProfile(dms_profile_t* profile) DmsInitCallback(&profile->callback); } +static inline void DMSDfxStatReset(){ + g_instance.dms_cxt.SSDFxStats.txnstatus_varcache_gets = 0; + g_instance.dms_cxt.SSDFxStats.txnstatus_hashcache_gets = 0; + g_instance.dms_cxt.SSDFxStats.txnstatus_network_io_gets = 0; + g_instance.dms_cxt.SSDFxStats.txnstatus_total_hcgets_time = 0; + g_instance.dms_cxt.SSDFxStats.txnstatus_total_niogets_time = 0; + g_instance.dms_cxt.SSDFxStats.txnstatus_total_evictions = 0; + g_instance.dms_cxt.SSDFxStats.txnstatus_total_eviction_refcnt = 0; +} + void DMSInit() { if (ss_dms_func_init() != DMS_SUCCESS) { @@ -409,6 +419,7 @@ void DMSInit() setDMSProfile(&profile); DMSInitLogger(); + DMSDfxStatReset(); g_instance.dms_cxt.log_timezone = u_sess->attr.attr_common.log_timezone; diff --git a/src/gausskernel/ddes/adapter/ss_transaction.cpp b/src/gausskernel/ddes/adapter/ss_transaction.cpp index 740085657..3561edf1d 100644 --- a/src/gausskernel/ddes/adapter/ss_transaction.cpp +++ b/src/gausskernel/ddes/adapter/ss_transaction.cpp @@ -32,6 +32,21 @@ #include "storage/sinvaladt.h" #include "replication/libpqsw.h" +static inline void txnstatusNetworkStats(uint64 timeDiff); +static inline void txnstatusHashStats(uint64 timeDiff); + +#define TxnStatusCalcStats(startTime, endTime, timeDiff, isHash) \ + do { \ + (void)INSTR_TIME_SET_CURRENT(endTime); \ + INSTR_TIME_SUBTRACT(endTime, startTime); \ + timeDiff = INSTR_TIME_GET_MICROSEC(endTime); \ + if (isHash) { \ + txnstatusHashStats((uint64)timeDiff); \ + } else { \ + txnstatusNetworkStats((uint64)timeDiff); \ + } \ + } while (0) + void SSStandbyGlobalInvalidSharedInvalidMessages(const SharedInvalidationMessage* msg, Oid tsid); Snapshot SSGetSnapshotData(Snapshot snapshot) @@ -82,6 +97,53 @@ static int SSTransactionIdGetCSN(dms_opengauss_xid_csn_t *dms_txn_info, dms_open return dms_request_opengauss_xid_csn(&dms_ctx, dms_txn_info, xid_csn_result); } +static inline void txnstatusNetworkStats(uint64 timeDiff) +{ + ereport(DEBUG1, (errmodule(MOD_SS_TXNSTATUS), + errmsg("SSTxnStatusCache niotimediff=%luus", timeDiff))); + g_instance.dms_cxt.SSDFxStats.txnstatus_network_io_gets++; + g_instance.dms_cxt.SSDFxStats.txnstatus_total_niogets_time += timeDiff; +} + +static inline void txnstatusHashStats(uint64 timeDiff) +{ + ereport(DEBUG1, (errmodule(MOD_SS_TXNSTATUS), + errmsg("SSTxnStatusCache hashtimediff=%luus", timeDiff))); + g_instance.dms_cxt.SSDFxStats.txnstatus_hashcache_gets++; + g_instance.dms_cxt.SSDFxStats.txnstatus_total_hcgets_time += timeDiff; +} + +/* + * Use TxnStatusCache regardless of the snapshot might result in erroneous visibility check. + * trustFrozen: whether frozen CSN can be used to determine visibility. if xid > snapshotxid, + * then still fetch from remote clogstatus cache. is clogstatus the most accurate? + */ +static inline bool SnapshotSatisfiesTSC(TransactionId transactionId, Snapshot snapshot, + bool isMvcc, bool *trustFrozen) +{ + if (snapshot == NULL || !isMvcc) { + return true; + } else { + if (IsVersionMVCCSnapshot(snapshot) || (snapshot->satisfies == SNAPSHOT_DECODE_MVCC)) { + return false; + } + if (IsMVCCSnapshot(snapshot) && !TransactionIdPrecedes(transactionId, snapshot->xmin)) { + *trustFrozen = false; + } + } + return true; +} + +static inline bool IsCommitSeqNoDefinitive(CommitSeqNo csn) +{ + return (COMMITSEQNO_IS_ABORTED(csn) || COMMITSEQNO_IS_COMMITTED(csn)); +} + +static inline bool IsClogStatusDefinitive(CLogXidStatus status) +{ + return (status != CLOG_XID_STATUS_IN_PROGRESS && status != CLOG_XID_STATUS_SUB_COMMITTED); +} + /* * xid -> csnlog status * is_committed: if true, then no need to fetch xid status from clog @@ -89,13 +151,20 @@ static int SSTransactionIdGetCSN(dms_opengauss_xid_csn_t *dms_txn_info, dms_open CommitSeqNo SSTransactionIdGetCommitSeqNo(TransactionId transactionId, bool isCommit, bool isMvcc, bool isNest, Snapshot snapshot, bool* sync) { + instr_time startTime; + instr_time endTime; + PgStat_Counter timeDiff = 0; + (void)INSTR_TIME_SET_CURRENT(startTime); + if ((snapshot == NULL || !IsVersionMVCCSnapshot(snapshot)) && TransactionIdEquals(transactionId, t_thrd.xact_cxt.cachedFetchCSNXid)) { + g_instance.dms_cxt.SSDFxStats.txnstatus_varcache_gets++; t_thrd.xact_cxt.latestFetchCSNXid = t_thrd.xact_cxt.cachedFetchCSNXid; t_thrd.xact_cxt.latestFetchCSN = t_thrd.xact_cxt.cachedFetchCSN; return t_thrd.xact_cxt.cachedFetchCSN; } if (!TransactionIdIsNormal(transactionId)) { + g_instance.dms_cxt.SSDFxStats.txnstatus_varcache_gets++; t_thrd.xact_cxt.latestFetchCSNXid = InvalidTransactionId; if (TransactionIdEquals(transactionId, BootstrapTransactionId) || TransactionIdEquals(transactionId, FrozenTransactionId)) { @@ -104,6 +173,23 @@ CommitSeqNo SSTransactionIdGetCommitSeqNo(TransactionId transactionId, bool isCo return COMMITSEQNO_ABORTED; } + CommitSeqNo cachedCSN = InvalidCommitSeqNo; + bool trustFrozen = false; + if (ENABLE_SS_TXNSTATUS_CACHE && SnapshotSatisfiesTSC(transactionId, snapshot, isMvcc, &trustFrozen)) { + bool cached = false; + uint32 hashcode = XidHashCode(&transactionId); + cached = TxnStatusCacheLookup(&transactionId, hashcode, &cachedCSN); + if (cached) { + Assert(IsCommitSeqNoDefinitive(cachedCSN)); + if (!COMMITSEQNO_IS_FROZEN(cachedCSN) || trustFrozen) { + TxnStatusCalcStats(startTime, endTime, timeDiff, true); +#ifndef USE_ASSERT_CHECKING + return cachedCSN; +#endif + } + } + } + CommitSeqNo csn = 0; // COMMITSEQNO_INPROGRESS by default CLogXidStatus clogstatus = CLOG_XID_STATUS_IN_PROGRESS; XLogRecPtr lsn = InvalidXLogRecPtr; @@ -151,17 +237,39 @@ CommitSeqNo SSTransactionIdGetCommitSeqNo(TransactionId transactionId, bool isCo } } while (true); - if (COMMITSEQNO_IS_COMMITTED(csn) || COMMITSEQNO_IS_ABORTED(csn)) { +#ifdef USE_ASSERT_CHECKING + /* DEBUG mode validates cache-networkIO consistency before returning cache */ + if (cachedCSN != InvalidCommitSeqNo && (!COMMITSEQNO_IS_FROZEN(cachedCSN) || trustFrozen)) { + Assert(((COMMITSEQNO_IS_COMMITTED(csn) && COMMITSEQNO_IS_COMMITTED(cachedCSN))) || + (COMMITSEQNO_IS_ABORTED(csn) && COMMITSEQNO_IS_ABORTED(cachedCSN))); + return cachedCSN; + } +#endif + + if (IsCommitSeqNoDefinitive(csn)) { t_thrd.xact_cxt.cachedFetchCSNXid = transactionId; t_thrd.xact_cxt.cachedFetchCSN = csn; + + if (ENABLE_SS_TXNSTATUS_CACHE && IsClogStatusDefinitive(clogstatus)) { + /* clogstat might be in-progress, therefore we want to be discreet */ + uint32 hashcode = XidHashCode(&transactionId); + LWLock *partitionLock = TxnStatusCachePartitionLock(hashcode); + LWLockAcquire(partitionLock, LW_EXCLUSIVE); + int ret = TxnStatusCacheInsert(&transactionId, hashcode, csn, clogstatus); + if (ret != GS_SUCCESS) { + ereport(PANIC, (errmsg("SSTxnStatusCache insert failed, xid=%lu.", transactionId))); + } + LWLockRelease(partitionLock); + } } - if (clogstatus != CLOG_XID_STATUS_IN_PROGRESS && clogstatus != CLOG_XID_STATUS_SUB_COMMITTED) { + if (IsClogStatusDefinitive(clogstatus)) { t_thrd.xact_cxt.cachedFetchXid = transactionId; t_thrd.xact_cxt.cachedFetchXidStatus = clogstatus; t_thrd.xact_cxt.cachedCommitLSN = lsn; } + TxnStatusCalcStats(startTime, endTime, timeDiff, false); return csn; } diff --git a/src/gausskernel/ddes/adapter/ss_txnstatus.cpp b/src/gausskernel/ddes/adapter/ss_txnstatus.cpp new file mode 100644 index 000000000..e5df2e53a --- /dev/null +++ b/src/gausskernel/ddes/adapter/ss_txnstatus.cpp @@ -0,0 +1,334 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * --------------------------------------------------------------------------------------- + * + * ss_txnstatus.cpp + * + * + * IDENTIFICATION + * src/gausskernel/ddes/adapter/ss_txnstatus.cpp + * + * --------------------------------------------------------------------------------------- + */ + +#include "ddes/dms/ss_txnstatus.h" +#include "postgres.h" +#include "utils/dynahash.h" +#include "access/transam.h" + +static inline int LRUIsFull(LRUQueue* queue) +{ + return queue->usedCount >= queue->maxSize; +} + +static inline int LRUIsEmpty(LRUQueue* queue) +{ + return queue->usedCount == 0; +} + +static inline unsigned int LRUGetUsage(LRUQueue* queue) +{ + return queue->usedCount; +} + +static inline unsigned int LRUGetMax(LRUQueue* queue) +{ + return queue->maxSize; +} + +static inline TxnStatusEntry* getTailLRUEntry(LRUQueue* queue) +{ + return queue->tail; +} + +static inline TxnStatusEntry* getHeadLRUEntry(LRUQueue* queue) +{ + return queue->head; +} + +static inline void updateEvictionRefcnt(TxnStatusEntry* entry) +{ + g_instance.dms_cxt.SSDFxStats.txnstatus_total_evictions++; + g_instance.dms_cxt.SSDFxStats.txnstatus_total_eviction_refcnt += entry->refcnt; + entry->refcnt = 0; +} + +/* + * Internal function; caller makes sure action is needed. + * Caller clears evicted entry's TxnStatus values if needed. + */ +static TxnStatusEntry* dequeueLRUEntry(LRUQueue* queue) +{ + if (unlikely(LRUIsEmpty(queue))) { + ereport(PANIC, (errmsg("SSTxnStatusCache LRU empty, used=%u, max=%u", + LRUGetUsage(queue), LRUGetMax(queue)))); + } + + TxnStatusEntry* temp = queue->tail; + if (temp->prev == NULL) { + queue->head = queue->tail = NULL; + } else { + queue->tail = temp->prev; + queue->tail->next = NULL; + } + queue->usedCount--; + + temp->prev = NULL; + temp->next = NULL; + temp->queueId = TXNSTATUS_CACHE_INVALID_QUEUEID; + updateEvictionRefcnt(temp); + return temp; +} + +/* + * Internal function, caller does eviction in advance if necessary. + * Caller validates entry's TxnStatus values. + */ +static void enqueueLRUEntry(LRUQueue* queue, TxnStatusEntry* entry) +{ + if (unlikely(LRUIsFull(queue))) { + ereport(PANIC, (errmsg("SSTxnStatusCache LRU is, usage=%u, max=%u", + LRUGetUsage(queue), LRUGetMax(queue)))); + } + + entry->prev = NULL; + entry->next = NULL; + entry->refcnt = 0; + if (LRUIsEmpty(queue)) { + queue->head = queue->tail = entry; + } else { + queue->head->prev = entry; + entry->next = queue->head; + queue->head = entry; + } + + entry->queueId = queue->id; + queue->usedCount++; +} + +/* caller makes sure referred entry is extant */ +static void referLRUEntry(LRUQueue* queue, TxnStatusEntry* entry) +{ + entry->refcnt++; + if (entry->prev == NULL) { + return; + } + + entry->prev->next = entry->next; + if (entry->next == NULL) { + queue->tail = entry->prev; + queue->tail->next = NULL; + } else { + entry->next->prev = entry->prev; + } + + entry->next = queue->head; + entry->prev = NULL; + entry->next->prev = entry; + queue->head = entry; +} + + +/* + * Compute the hash code associated with a Xid. + * + * To avoid unnecessary recomputations of the hash code, we try to do this + * just once per procedure, and then pass it around as needed. Aside from + * passing the hashcode to hash_search_with_hash_value(), we can extract + * the lock partition number from the hashcode. + */ +uint32 XidHashCode(const TransactionId *xid) +{ + return get_hash_value(t_thrd.dms_cxt.SSTxnStatusHash, (const void *)xid); +} + +/* + * Init TxnStatusLRU. + */ +static void initTxnStatusLRU(uint32 size) +{ + bool lruInited = false; + uint32 lruSize = size / NUM_TXNSTATUS_CACHE_PARTITIONS; + ereport(DEBUG1, (errmodule(MOD_SS_TXNSTATUS), + errmsg("SSTxnStatusCache totalsz=%u, partcnt=LRUcnt=%u, LRUsz=%um.", + size, NUM_TXNSTATUS_CACHE_PARTITIONS, lruSize))); + + t_thrd.dms_cxt.SSTxnStatusLRU = (LRUQueue *)CACHELINEALIGN(ShmemInitStruct("SS Xid LRU cache", + NUM_TXNSTATUS_CACHE_PARTITIONS * sizeof(LRUQueue) + PG_CACHE_LINE_SIZE, &lruInited)); + if (!lruInited) { + for (unsigned int i = 0; i < NUM_TXNSTATUS_CACHE_PARTITIONS; i++) { + LRUQueue *queue = &t_thrd.dms_cxt.SSTxnStatusLRU[i]; + queue->usedCount = 0; + queue->maxSize = lruSize; + queue->head = NULL; + queue->tail = NULL; + queue->id = i; + } + } +} + +/* + * Initialize shmem hash table for mapping txninfo + */ +static void InitTxnStatusTable(unsigned int maxsize) +{ + HASHCTL hctl; + int hflags; + long initSize; + + /* + * Init max size to request for TxnStatus hashtables. + */ + initSize = maxsize; + + /* + * Allocate hash table for TxnStatusEntry structs. + */ + errno_t rc = memset_s(&hctl, sizeof(hctl), 0, sizeof(hctl)); + securec_check(rc, "", ""); + hctl.keysize = sizeof(TransactionId); + hctl.entrysize = sizeof(TxnStatusEntry); + hctl.hash = tag_hash; + hctl.num_partitions = NUM_TXNSTATUS_CACHE_PARTITIONS; + hflags = (HASH_ELEM | HASH_FUNCTION | HASH_PARTITION); + t_thrd.dms_cxt.SSTxnStatusHash = ShmemInitHash("TxnStatus hash", + initSize, maxsize, &hctl, hflags); +} + +void SSInitTxnStatusCache() +{ + if (!ENABLE_DMS || !ENABLE_SS_TXNSTATUS_CACHE) { + return; + } + + Assert(NUM_TXNSTATUS_CACHE_ENTRIES % NUM_TXNSTATUS_CACHE_PARTITIONS == 0); + initTxnStatusLRU(NUM_TXNSTATUS_CACHE_ENTRIES); + InitTxnStatusTable(NUM_TXNSTATUS_CACHE_ENTRIES); +} + +/* + * TxnStatusCacheInsert + * Look up the hashtable entry for given XID, which may not exist + * + * Caller does not need to hold partlock; this function decides locking S or X. + */ +int TxnStatusCacheInsert(const TransactionId *xid, uint32 hashcode, CommitSeqNo csn, CLogXidStatus clogStatus) +{ + uint32 evictHash; + bool found = false; + uint32 ret = GS_SUCCESS; + TxnStatusEntry *insertEntry = NULL; + TxnStatusEntry *evictEntry = NULL; + HTAB* hashp = t_thrd.dms_cxt.SSTxnStatusHash; + const uint32 queueId = TxnStatusCachePartitionId(hashcode); + LRUQueue* queue = &t_thrd.dms_cxt.SSTxnStatusLRU[queueId]; + bool triggered = LRUGetUsage(queue) * 1.0 / LRUGetMax(queue) >= TXNSTATUS_CACHE_LRU_FFACTOR; + + /* + * Trigger LRU-based entry eviction on-demand: peek tail, rm hash, and then rm LRU. + * Concurrency would occur were multiple xid insertions happen to evict the same xid, + * subpartition of which is lockfree, resulting in either the 2nd eviction failure during + * htable deletion, or both succeeds, causing the later evicts an extra LRU entry. + * Eviction concurreny is no longer worried as LRU partlock now guarantees consistency. + */ + if (triggered) { + evictEntry = dequeueLRUEntry(queue); + evictHash = XidHashCode(&evictEntry->xid); + ereport(DEBUG1, (errmodule(MOD_SS_TXNSTATUS), + errmsg("SSTxnStatusCache eviction xid=%lu, queue=%u, queue_usage=%u of %u", + (uint64)evictEntry->xid, queueId, queue->usedCount, queue->maxSize))); + ret = TxnStatusCacheDelete(&evictEntry->xid, evictHash); + if (ret != GS_SUCCESS) { + ereport(PANIC, (errmodule(MOD_SS_TXNSTATUS), + errmsg("SSTxnStatusCache discrepancy, xid=%lu in Q=%u, not in hash", + (uint64)evictEntry->xid, queueId))); + } + } + + /* + * refer or insert an entry associated with this xid + */ + insertEntry = (TxnStatusEntry *)hash_search_with_hash_value(hashp, + (const void *)xid, hashcode, HASH_ENTER_NULL, &found); + if (insertEntry == NULL) { + ereport(PANIC, (errmsg("Insert SSTxnStatusCache OOM, check usage"))); + } + + /* + * if it is a new TxnStatusEntry, initialize it and push to LRU; + * if entry exists, double check and boost its position in LRU + */ + if (!found) { + insertEntry->csn = csn; + insertEntry->clogStatus = clogStatus; + enqueueLRUEntry(queue, insertEntry); + } else { + referLRUEntry(queue, insertEntry); + } + + return GS_SUCCESS; +} + +/* + * TxnStatusCacheLookup + * Look up the hashtable entry for given XID, which may not exist + * + * Caller does not need to hold partlock; this function decides locking S or X. + */ +bool TxnStatusCacheLookup(TransactionId *xid, uint32 hashcode, CommitSeqNo *cached) +{ + TxnStatusEntry *entry = NULL; + LWLock* partlock = NULL; + const uint32 queueId = TxnStatusCachePartitionId(hashcode); + LRUQueue* queue = &t_thrd.dms_cxt.SSTxnStatusLRU[queueId]; + partlock = TxnStatusCachePartitionLock(hashcode); + + LWLockAcquire(partlock, LW_EXCLUSIVE); + entry = (TxnStatusEntry *)hash_search_with_hash_value(t_thrd.dms_cxt.SSTxnStatusHash, + (const void *)xid, hashcode, HASH_FIND, NULL); + + if (entry != NULL) { + if (entry->queueId != queueId) { + Assert(0); /* should not happen */ + } else { + *cached = entry->csn; + referLRUEntry(queue, entry); + } + } else { + ereport(DEBUG1, (errmodule(MOD_SS_TXNSTATUS), + errmsg("SSTxnStatusCache xid=%lu not extant or lock timeout", (uint64)*xid))); + LWLockRelease(partlock); + return false; + } + + LWLockRelease(partlock); + return true; +} + +/* + * TxnStatusCacheDelete + * Delete the hashtable entry for given tag which might not exist + * Caller must hold respective partlock. + * Caller does repective eviction. + */ +int TxnStatusCacheDelete(TransactionId *xid, uint32 hashcode) +{ + TxnStatusEntry *txnStatusEntry = NULL; + txnStatusEntry = (TxnStatusEntry *)hash_search_with_hash_value(t_thrd.dms_cxt.SSTxnStatusHash, + (const void *)xid, hashcode, HASH_REMOVE, NULL); + if (txnStatusEntry == NULL) { + return GS_ERROR; + } + return GS_SUCCESS; +} diff --git a/src/gausskernel/process/threadpool/knl_thread.cpp b/src/gausskernel/process/threadpool/knl_thread.cpp index 8bcf7571a..6bae653ad 100755 --- a/src/gausskernel/process/threadpool/knl_thread.cpp +++ b/src/gausskernel/process/threadpool/knl_thread.cpp @@ -1709,6 +1709,8 @@ static void knl_t_dms_context_init(knl_t_dms_context *dms_cxt) errno_t rc = memset_s(dms_cxt->msg_backup, sizeof(dms_cxt->msg_backup), 0, sizeof(dms_cxt->msg_backup)); securec_check(rc, "\0", "\0"); dms_cxt->flush_copy_get_page_failed = false; + dms_cxt->SSTxnStatusHash = NULL; + dms_cxt->SSTxnStatusLRU = NULL; } static void knl_t_ondemand_xlog_copy_context_init(knl_t_ondemand_xlog_copy_context *ondemand_xlog_copy_cxt) diff --git a/src/gausskernel/storage/ipc/ipci.cpp b/src/gausskernel/storage/ipc/ipci.cpp index 0f1a23838..86c90b166 100644 --- a/src/gausskernel/storage/ipc/ipci.cpp +++ b/src/gausskernel/storage/ipc/ipci.cpp @@ -82,6 +82,7 @@ #include "replication/dcf_replication.h" #include "commands/verify.h" #include "storage/cfs/cfs_buffers.h" +#include "ddes/dms/ss_txnstatus.h" /* we use semaphore not LWLOCK, because when thread InitGucConfig, it does not get a t_thrd.proc */ pthread_mutex_t gLocaleMutex = PTHREAD_MUTEX_INITIALIZER; @@ -319,6 +320,8 @@ void CreateSharedMemoryAndSemaphores(bool makePrivate, int port) * Set up predicate lock manager */ InitPredicateLocks(); + + SSInitTxnStatusCache(); } /* diff --git a/src/gausskernel/storage/lmgr/lwlock.cpp b/src/gausskernel/storage/lmgr/lwlock.cpp index d00e31e84..7d852b5cd 100644 --- a/src/gausskernel/storage/lmgr/lwlock.cpp +++ b/src/gausskernel/storage/lmgr/lwlock.cpp @@ -197,7 +197,8 @@ static const char *BuiltinTrancheNames[] = { "ReplicationOriginLock", "AuditIndextblLock", "PCABufferContentLock", - "XlogTrackPartLock" + "XlogTrackPartLock", + "SSTxnStatusCachePartLock" }; static void RegisterLWLockTranches(void); @@ -440,6 +441,9 @@ int NumLWLocks(void) /* for xlog track hash table */ numLocks += NUM_XLOG_TRACK_PARTITIONS; + /* for ss txnstatus hash table */ + numLocks += NUM_TXNSTATUS_CACHE_PARTITIONS; + /* * Add any requested by loadable modules; for backwards-compatibility * reasons, allocate at least NUM_USER_DEFINED_LWLOCKS of them even if @@ -654,6 +658,10 @@ static void InitializeLWLocks(int numLocks) LWLockInitialize(&lock->lock, LWTRANCHE_XLOG_TRACK_PARTITION); } + for (id = 0; id < NUM_TXNSTATUS_CACHE_PARTITIONS; id++, lock++) { + LWLockInitialize(&lock->lock, LWTRANCHE_SS_TXNSTATUS_PARTITION); + } + Assert((lock - t_thrd.shemem_ptr_cxt.mainLWLockArray) == NumFixedLWLocks); for (id = NumFixedLWLocks; id < numLocks; id++, lock++) { diff --git a/src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback-post_catalog_maindb_92_906.sql b/src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback-post_catalog_maindb_92_906.sql new file mode 100644 index 000000000..d1f556cdc --- /dev/null +++ b/src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback-post_catalog_maindb_92_906.sql @@ -0,0 +1 @@ +DROP FUNCTION IF EXISTS pg_catalog.ss_txnstatus_cache_stat() CASCADE; diff --git a/src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback-post_catalog_otherdb_92_906.sql b/src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback-post_catalog_otherdb_92_906.sql new file mode 100644 index 000000000..d1f556cdc --- /dev/null +++ b/src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback-post_catalog_otherdb_92_906.sql @@ -0,0 +1 @@ +DROP FUNCTION IF EXISTS pg_catalog.ss_txnstatus_cache_stat() CASCADE; diff --git a/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_906.sql b/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_906.sql new file mode 100644 index 000000000..a720071e5 --- /dev/null +++ b/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_906.sql @@ -0,0 +1,15 @@ +DROP FUNCTION IF EXISTS pg_catalog.ss_txnstatus_cache_stat() CASCADE; +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 8888; +CREATE FUNCTION pg_catalog.ss_txnstatus_cache_stat( + OUT vcache_gets bigint, + OUT hcache_gets bigint, + OUT nio_gets bigint, + OUT avg_hcache_gettime_us float8, + OUT avg_nio_gettime_us float8, + OUT cache_hit_rate float8, + OUT hcache_eviction bigint, + OUT avg_eviction_refcnt float8 +) +RETURNS SETOF record +LANGUAGE internal STABLE NOT FENCED NOT SHIPPABLE ROWS 100 +AS 'ss_txnstatus_cache_stat'; diff --git a/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_906.sql b/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_906.sql new file mode 100644 index 000000000..a720071e5 --- /dev/null +++ b/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_906.sql @@ -0,0 +1,15 @@ +DROP FUNCTION IF EXISTS pg_catalog.ss_txnstatus_cache_stat() CASCADE; +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 8888; +CREATE FUNCTION pg_catalog.ss_txnstatus_cache_stat( + OUT vcache_gets bigint, + OUT hcache_gets bigint, + OUT nio_gets bigint, + OUT avg_hcache_gettime_us float8, + OUT avg_nio_gettime_us float8, + OUT cache_hit_rate float8, + OUT hcache_eviction bigint, + OUT avg_eviction_refcnt float8 +) +RETURNS SETOF record +LANGUAGE internal STABLE NOT FENCED NOT SHIPPABLE ROWS 100 +AS 'ss_txnstatus_cache_stat'; diff --git a/src/include/ddes/dms/ss_common_attr.h b/src/include/ddes/dms/ss_common_attr.h index cefce4989..c04d0a63a 100644 --- a/src/include/ddes/dms/ss_common_attr.h +++ b/src/include/ddes/dms/ss_common_attr.h @@ -35,10 +35,12 @@ #define ENABLE_DMS false #define ENABLE_REFORM false #define ENABLE_VERIFY_PAGE_VERSION false +#define ENABLE_SS_TXNSTATUS_CACHE false #else #define ENABLE_DMS (g_instance.attr.attr_storage.dms_attr.enable_dms && !IsInitdb) #define ENABLE_REFORM (g_instance.attr.attr_storage.dms_attr.enable_reform) #define ENABLE_VERIFY_PAGE_VERSION (g_instance.attr.attr_storage.dms_attr.enable_verify_page) +#define ENABLE_SS_TXNSTATUS_CACHE (ENABLE_DMS && g_instance.attr.attr_storage.dms_attr.txnstatus_cache_size > 0) #endif #define SS_REFORM_REFORMER \ @@ -237,5 +239,15 @@ typedef enum SSOndemandRequestRedoStatus { ONDEMAND_REDO_TIMEOUT } SSOndemandRequestRedoStatus; +/* consider DFX stats reset were node role to change */ +typedef struct ss_dfx_stats_t { + uint64 txnstatus_varcache_gets; + uint64 txnstatus_hashcache_gets; + uint64 txnstatus_network_io_gets; + uint64 txnstatus_total_niogets_time; + uint64 txnstatus_total_hcgets_time; + uint64 txnstatus_total_evictions; + uint64 txnstatus_total_eviction_refcnt; +} ss_dfx_stats_t; #endif diff --git a/src/include/ddes/dms/ss_txnstatus.h b/src/include/ddes/dms/ss_txnstatus.h new file mode 100644 index 000000000..47c776ffc --- /dev/null +++ b/src/include/ddes/dms/ss_txnstatus.h @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * --------------------------------------------------------------------------------------- + * + * ss_txnstatus.h + * + * + * IDENTIFICATION + * src/gausskernel/ddes/adapter/ss_txnstatus.h + * + * --------------------------------------------------------------------------------------- + */ + +#ifndef SS_TXNSTATUS_H +#define SS_TXNSTATUS_H + +#include "postgres.h" +#include "c.h" +#include "access/clog.h" + +typedef struct TxnStatusEntry { + TransactionId xid; + CommitSeqNo csn; + CLogXidStatus clogStatus; + TxnStatusEntry* prev; + TxnStatusEntry* next; + volatile uint32 queueId; + uint32 refcnt; +} TxnStatusEntry; + +typedef struct LRUQueue { + unsigned int id; + unsigned int usedCount; + unsigned int maxSize; + TxnStatusEntry *head; + TxnStatusEntry *tail; +} LRUQueue; + +#define TXNSTATUS_CACHE_LRU_FFACTOR 0.9 +#define TXNSTATUS_CACHE_INVALID_QUEUEID (NUM_TXNSTATUS_CACHE_PARTITIONS + 1) +#define NUM_TXNSTATUS_CACHE_ENTRIES (g_instance.attr.attr_storage.dms_attr.txnstatus_cache_size) +#define TxnStatusCachePartitionId(hashcode) ((hashcode) % NUM_TXNSTATUS_CACHE_PARTITIONS) +#define TxnStatusCachePartitionLock(hashcode) \ + (&t_thrd.shemem_ptr_cxt.mainLWLockArray[FirstTxnStatusCacheLock + TxnStatusCachePartitionId(hashcode)].lock) +#define TXNSTATUS_LWLOCK_ACQUIRE(hashcode, lockmode) \ + ((void)LWLockAcquire(TxnStatusCachePartitionLock(hashcode), lockmode)) +#define TXNSTATUS_LWLOCK_RELEASE(hashcode) (LWLockRelease(TxnStatusCachePartitionLock(hashcode))) + +uint32 XidHashCode(const TransactionId *xid); +void SSInitTxnStatusCache(); +int TxnStatusCacheInsert(const TransactionId *xid, uint32 hashcode, CommitSeqNo csn, CLogXidStatus clogStatus); +bool TxnStatusCacheLookup(TransactionId *xid, uint32 hashcode, CommitSeqNo *cached); +int TxnStatusCacheDelete(TransactionId *xid, uint32 hashcode); + +#endif /* SS_TXNSTATUS_H */ \ No newline at end of file diff --git a/src/include/knl/knl_guc/knl_instance_attr_storage.h b/src/include/knl/knl_guc/knl_instance_attr_storage.h index 003a59bf9..8d17c7e67 100755 --- a/src/include/knl/knl_guc/knl_instance_attr_storage.h +++ b/src/include/knl/knl_guc/knl_instance_attr_storage.h @@ -124,6 +124,7 @@ typedef struct knl_instance_attr_dms { int32 sslog_backup_file_count; int32 sslog_max_file_size; //Unit:KB int parallel_thread_num; + int32 txnstatus_cache_size; } knl_instance_attr_dms; typedef struct knl_instance_attr_storage { diff --git a/src/include/knl/knl_instance.h b/src/include/knl/knl_instance.h index 914bbf05f..3b41f74c4 100755 --- a/src/include/knl/knl_instance.h +++ b/src/include/knl/knl_instance.h @@ -1214,6 +1214,7 @@ typedef struct knl_g_dms_context { bool dw_init; char dmsInstAddr[MAX_REPLNODE_NUM][DMS_MAX_IP_LEN]; char conninfo[MAXPGPATH]; + ss_dfx_stats_t SSDFxStats; } knl_g_dms_context; typedef struct knl_instance_context { diff --git a/src/include/knl/knl_thread.h b/src/include/knl/knl_thread.h index 524c1524f..78fec000f 100755 --- a/src/include/knl/knl_thread.h +++ b/src/include/knl/knl_thread.h @@ -75,6 +75,7 @@ #include "catalog/pg_subscription.h" #include "port/pg_crc32c.h" #include "ddes/dms/ss_common_attr.h" +#include "ddes/dms/ss_txnstatus.h" #define MAX_PATH_LEN 1024 extern const int g_reserve_param_num; @@ -3356,6 +3357,8 @@ typedef struct knl_t_dms_context { int file_size; /* initialized as pg_internal.init file size, will decrease after read */ char msg_backup[24]; // 24 is sizeof mes_message_head_t bool flush_copy_get_page_failed; //used in flush copy + HTAB* SSTxnStatusHash; + LRUQueue* SSTxnStatusLRU; } knl_t_dms_context; typedef struct knl_t_ondemand_xlog_copy_context { diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h index 8d049d420..a4ab2eb9d 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h @@ -37,6 +37,7 @@ /***************************************************************************** * Backend version and inplace upgrade staffs *****************************************************************************/ +extern const uint32 TXNSTATUS_CACHE_DFX_VERSION_NUM; extern const uint32 TIMESCALE_DB_VERSION_NUM; extern const uint32 NBTREE_INSERT_OPTIMIZATION_VERSION_NUM; extern const uint32 NBTREE_DEDUPLICATION_VERSION_NUM; diff --git a/src/include/storage/lock/lwlock.h b/src/include/storage/lock/lwlock.h index 92858f0dd..94db36d51 100644 --- a/src/include/storage/lock/lwlock.h +++ b/src/include/storage/lock/lwlock.h @@ -148,6 +148,9 @@ const struct LWLOCK_PARTITION_DESC LWLockPartInfo[] = { /* Number of standby statement hsitory needed */ #define NUM_STANDBY_STMTHIST_PARTITIONS 2 +/* Number of partitions of the txnstatus mapping hashtable */ +#define NUM_TXNSTATUS_CACHE_PARTITIONS 256 + /* * WARNING---Please keep the order of LWLockTrunkOffset and BuiltinTrancheIds consistent!!! */ @@ -194,8 +197,11 @@ const struct LWLOCK_PARTITION_DESC LWLockPartInfo[] = { /* standby statement history */ #define FirstStandbyStmtHistLock (FirstGPRCMappingLock + NUM_GPRC_PARTITIONS) #define FirstXlogTrackLock (FirstStandbyStmtHistLock + NUM_STANDBY_STMTHIST_PARTITIONS) + +/* txn status cache */ +#define FirstTxnStatusCacheLock (FirstXlogTrackLock + NUM_XLOG_TRACK_PARTITIONS) /* must be last: */ -#define NumFixedLWLocks (FirstXlogTrackLock + NUM_XLOG_TRACK_PARTITIONS) +#define NumFixedLWLocks (FirstTxnStatusCacheLock + NUM_TXNSTATUS_CACHE_PARTITIONS) /* * WARNING----Please keep BuiltinTrancheIds and BuiltinTrancheNames consistent!!! * @@ -275,6 +281,7 @@ enum BuiltinTrancheIds LWTRANCHE_AUDIT_INDEX_WAIT, LWTRANCHE_PCA_BUFFER_CONTENT, LWTRANCHE_XLOG_TRACK_PARTITION, + LWTRANCHE_SS_TXNSTATUS_PARTITION, /* * Each trancheId above should have a corresponding item in BuiltinTrancheNames; */ diff --git a/src/include/utils/be_module.h b/src/include/utils/be_module.h index a2a0f3fee..d7db7a9c4 100755 --- a/src/include/utils/be_module.h +++ b/src/include/utils/be_module.h @@ -149,6 +149,7 @@ enum ModuleId { MOD_GPI, /* debug info for global partition index */ MOD_PARTITION, MOD_SRF, /* debug info for SRF */ + MOD_SS_TXNSTATUS, /* debug info for SS TXNSTATUS */ /* * Add your module id above. diff --git a/src/test/regress/expected/opr_sanity.out b/src/test/regress/expected/opr_sanity.out index 8c90f77c6..bacd40ff8 100644 --- a/src/test/regress/expected/opr_sanity.out +++ b/src/test/regress/expected/opr_sanity.out @@ -2719,6 +2719,7 @@ WHERE d.classoid IS NULL AND p1.oid <= 9999 order by 1; 7777 | sysdate 7998 | set_working_grand_version_num_manually 8050 | datalength + 8888 | ss_txnstatus_cache_stat 9004 | smalldatetime_in 9006 | smalldatetime_out 9007 | smalldatetime_recv diff --git a/src/test/regress/expected/single_node_opr_sanity.out b/src/test/regress/expected/single_node_opr_sanity.out index 959b823fb..443d9c477 100755 --- a/src/test/regress/expected/single_node_opr_sanity.out +++ b/src/test/regress/expected/single_node_opr_sanity.out @@ -1552,6 +1552,7 @@ WHERE d.classoid IS NULL AND p1.oid <= 9999 order by 1; 8703 | array_except 8704 | array_except_distinct 8852 | pg_event_trigger_ddl_commands + 8888 | ss_txnstatus_cache_stat 9004 | smalldatetime_in 9006 | smalldatetime_out 9007 | smalldatetime_recv diff --git a/src/test/regress/output/recovery_2pc_tools.source b/src/test/regress/output/recovery_2pc_tools.source index c491ca40a..76c79865f 100644 --- a/src/test/regress/output/recovery_2pc_tools.source +++ b/src/test/regress/output/recovery_2pc_tools.source @@ -652,6 +652,7 @@ select name,vartype,unit,min_val,max_val from pg_settings where name <> 'qunit_c ss_scrlock_server_port | integer | | 0 | 65535 ss_scrlock_worker_bind_core | string | | | ss_scrlock_worker_count | integer | | 2 | 16 + ss_txnstatus_cache_size | integer | | 0 | 524288 ss_work_thread_count | integer | | 16 | 128 standard_conforming_strings | bool | | | standby_shared_buffers_fraction | real | | 0.1 | 1 From 8c073974450efa5237261cb1513660fbf76584c1 Mon Sep 17 00:00:00 2001 From: gbzhangkai Date: Tue, 25 Jul 2023 17:12:11 +0800 Subject: [PATCH 051/304] =?UTF-8?q?=E6=97=A0=E7=AC=A6=E5=8F=B7=E7=B1=BB?= =?UTF-8?q?=E5=9E=8B=E6=AF=94=E8=BE=83=E4=B8=8D=E5=BD=93=E5=AF=BC=E8=87=B4?= =?UTF-8?q?=E5=88=A4=E6=96=AD=E6=9D=A1=E4=BB=B6=E5=86=97=E4=BD=99=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- contrib/gauss_connector/deparse.cpp | 2 +- contrib/gauss_connector/gc_fdw.cpp | 2 +- contrib/pg_standby/pg_standby.cpp | 2 +- contrib/security_plugin/gs_policy_object_types.cpp | 2 +- contrib/security_plugin/masking.cpp | 4 ++-- src/common/backend/utils/cache/plancache.cpp | 2 +- src/gausskernel/cbb/instruments/statement/instr_statement.cpp | 4 ++-- src/gausskernel/storage/access/transam/xlog.cpp | 4 ++-- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/contrib/gauss_connector/deparse.cpp b/contrib/gauss_connector/deparse.cpp index 5434b3c89..2b0d5f538 100644 --- a/contrib/gauss_connector/deparse.cpp +++ b/contrib/gauss_connector/deparse.cpp @@ -2798,7 +2798,7 @@ Plan* deparse_agg_node(Plan* agg, PlannerInfo* root) ForeignScan* fscan = (ForeignScan*)agg->lefttree; - if (fscan->scan.scanrelid <= 0 || (int)fscan->scan.scanrelid >= root->simple_rel_array_size) + if (fscan->scan.scanrelid == 0 || (int)fscan->scan.scanrelid >= root->simple_rel_array_size) return agg; RelOptInfo* scanrel = root->simple_rel_array[fscan->scan.scanrelid]; diff --git a/contrib/gauss_connector/gc_fdw.cpp b/contrib/gauss_connector/gc_fdw.cpp index 15bcd2d80..51ae5ecbd 100644 --- a/contrib/gauss_connector/gc_fdw.cpp +++ b/contrib/gauss_connector/gc_fdw.cpp @@ -327,7 +327,7 @@ static void gcGetForeignRelSize(PlannerInfo* root, RelOptInfo* baserel, Oid fore gcfdw_fetch_remote_table_info(pgxc_handle, fpinfo->table, remote_info, PGFDW_GET_TABLE_INFO); Assert(remote_info->snapsize > 0); - if (remote_info->snapsize <= 0) { + if (remote_info->snapsize == 0) { ereport(ERROR, (errcode(ERRCODE_NO_DATA_FOUND), errmsg("cooperation analysis: not receive the snapshot from remote cluster"))); diff --git a/contrib/pg_standby/pg_standby.cpp b/contrib/pg_standby/pg_standby.cpp index 5ae323673..ed33751ba 100644 --- a/contrib/pg_standby/pg_standby.cpp +++ b/contrib/pg_standby/pg_standby.cpp @@ -326,7 +326,7 @@ static bool SetWALFileNameForCleanup(void) if (keepfiles > 0) { sscanf(nextWALFileName, "%08X%08X%08X", &tli, &log, &seg); - if (tli > 0 && log >= 0 && seg > 0) { + if (tli > 0 && seg > 0) { log_diff = keepfiles / MaxSegmentsPerLogFile; seg_diff = keepfiles % MaxSegmentsPerLogFile; if (seg_diff > seg) { diff --git a/contrib/security_plugin/gs_policy_object_types.cpp b/contrib/security_plugin/gs_policy_object_types.cpp index 42bf894aa..006dcb7ce 100644 --- a/contrib/security_plugin/gs_policy_object_types.cpp +++ b/contrib/security_plugin/gs_policy_object_types.cpp @@ -260,7 +260,7 @@ void PolicyLabelItem::init(const PolicyLabelItem &arg) m_object = arg.m_object; errno_t rc = memset_s(m_column, sizeof(m_column), 0, sizeof(m_column)); securec_check(rc, "\0", "\0"); - if (arg.m_column != NULL && strlen(arg.m_column) > 0) + if (strlen(arg.m_column) > 0) { rc = snprintf_s(m_column, sizeof(m_column), strlen(arg.m_column), "%s", arg.m_column); securec_check_ss(rc, "\0", "\0"); diff --git a/contrib/security_plugin/masking.cpp b/contrib/security_plugin/masking.cpp index d91813fc3..fec23dc1f 100644 --- a/contrib/security_plugin/masking.cpp +++ b/contrib/security_plugin/masking.cpp @@ -805,7 +805,7 @@ static Node* create_udf_function(ParseState *pstate, Var* var, Oid funcid, maski Oid rescollid = 100; /* OID of collation, or InvalidOid if none */ PG_TRY(); { - if (funcid <= 0) { + if (funcid == 0) { return NULL; } Oid rettype = var->vartype; @@ -1393,7 +1393,7 @@ bool handle_masking(List *targetList, ParseState *pstate, const policy_set *poli /* Shuffle masking columns can only select directly with out other operations */ parser_target_entry(pstate, old_tle, policy_ids, &masking_result, rtable, true); } - if (masking_result.size() <= 0) { + if (masking_result.size() == 0) { return false; } if (strlen(t_thrd.security_policy_cxt.prepare_stmt_name) > 0) { diff --git a/src/common/backend/utils/cache/plancache.cpp b/src/common/backend/utils/cache/plancache.cpp index 78d1dbd34..d4de8f79c 100644 --- a/src/common/backend/utils/cache/plancache.cpp +++ b/src/common/backend/utils/cache/plancache.cpp @@ -303,7 +303,7 @@ CachedPlanSource* CreateCachedPlan(Node* raw_parse_tree, const char* query_strin if (enable_pbe_gpc) { plansource->gpc.status.ShareInit(); } else if (ENABLE_CN_GPC && enable_spi_gpc) { - Assert(u_sess->SPI_cxt._current->plan_id >= 0 && u_sess->SPI_cxt._current->visit_id >= 0); + Assert(u_sess->SPI_cxt._current->plan_id >= 0); plansource->gpc.status.ShareInit(); plansource->spi_signature.spi_key = u_sess->SPI_cxt._current->spi_hash_key; plansource->spi_signature.func_oid = u_sess->SPI_cxt._current->func_oid; diff --git a/src/gausskernel/cbb/instruments/statement/instr_statement.cpp b/src/gausskernel/cbb/instruments/statement/instr_statement.cpp index afccc4b6b..9573f4b84 100755 --- a/src/gausskernel/cbb/instruments/statement/instr_statement.cpp +++ b/src/gausskernel/cbb/instruments/statement/instr_statement.cpp @@ -1699,7 +1699,7 @@ static int stmt_compare_wait_events(const void *a, const void *b) void decode_stmt_wait_events(StringInfo resultBuf, const char *details, uint32 total_len, bool *is_valid_record, bool pretty) { - if (total_len <= 0) { + if (total_len == 0) { *is_valid_record = false; return; } @@ -1771,7 +1771,7 @@ void decode_stmt_wait_events(StringInfo resultBuf, const char *details, void decode_statement_detail(StringInfo resultBuf, const char *details, uint32 total_len, bool *is_valid_record, bool pretty) { - if (total_len <= 0) { + if (total_len == 0) { *is_valid_record = false; return; } diff --git a/src/gausskernel/storage/access/transam/xlog.cpp b/src/gausskernel/storage/access/transam/xlog.cpp index 3d6dc1d24..4e44adad7 100755 --- a/src/gausskernel/storage/access/transam/xlog.cpp +++ b/src/gausskernel/storage/access/transam/xlog.cpp @@ -12930,7 +12930,7 @@ static void KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo, XLogRecPtr curIns XLByteToSeg(keep, slotSegNo); - if (slotSegNo <= 0) { + if (slotSegNo == 0) { /* segno = 1 show all file should be keep */ segno = 1; ereport(LOG, (errmsg("keep all the xlog segments, because the minimal replication slot segno " @@ -12948,7 +12948,7 @@ static void KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo, XLogRecPtr curIns XLogSegNo slotSegNo; XLByteToSeg(xlogcopystartptr, slotSegNo); - if (slotSegNo <= 0) { + if (slotSegNo == 0) { /* segno = 1 show all file should be keep */ segno = 1; ereport(LOG, (errmsg("keep all the xlog segments, because there is a full-build task in the backend, " From 0232af2917809d716ed781268df9bdaa0d1b6bc7 Mon Sep 17 00:00:00 2001 From: z00793368 Date: Tue, 25 Jul 2023 19:36:36 +0800 Subject: [PATCH 052/304] add keymgr module Offering: openGaussDev More detail: add keymgr module Match-id-7ee0d8b52de7787a980066a17defd5e2acc1bfab --- .../backend/client_logic/client_logic.cpp | 2 +- src/common/backend/parser/gram.y | 2 +- src/common/interfaces/libpq/CMakeLists.txt | 28 +- src/common/interfaces/libpq/Makefile | 21 +- src/common/interfaces/libpq/cl_state.cpp | 4 + .../cmk_entity_manager_hooks/Makefile | 77 -- .../cmk_entity_manager_hooks/cmkem_comm.cpp | 720 ------------ .../cmkem_comm_http.cpp | 363 ------ .../cmkem_comm_http.h | 149 --- .../kms_restful_temp.ini | 102 -- .../reg_cmkem_manager_main.cpp | 103 -- .../reg_hook_frame.cpp | 190 ---- .../cmk_entity_manager_hooks/reg_hook_frame.h | 85 -- .../register_gram_check.cpp | 67 -- .../register_gram_check.h | 34 - .../register_huawei_kms.cpp | 994 ---------------- .../register_huawei_kms.h | 84 -- .../register_local_kms.cpp | 1006 ----------------- .../encryption_hooks/encrypt_decrypt.cpp | 522 --------- .../encryption_column_hook_executor.cpp | 68 +- .../encryption_column_hook_executor.h | 4 +- .../encryption_global_hook_executor.cpp | 97 +- .../interfaces/libpq/frontend_parser/gram.y | 2 +- .../jdbc/client_logic_jni/client_logic_jni.h | 1 - .../org_postgresql_jdbc_ClientLogicImpl.cpp | 2 +- src/gausskernel/security/CMakeLists.txt | 2 + src/gausskernel/security/Makefile | 2 +- .../security/keymgr/CMakeLists.txt | 90 ++ src/gausskernel/security/keymgr/Makefile | 41 + .../security/keymgr/api/security_key_adpt.cpp | 418 +++++++ .../security/keymgr/comm/security_aksk.cpp | 238 ++++ .../security/keymgr/comm/security_encode.cpp | 227 ++++ .../security/keymgr/comm/security_error.cpp | 99 ++ .../security/keymgr/comm/security_http.cpp | 377 ++++++ .../keymgr/comm/security_httpscan.cpp | 281 +++++ .../security/keymgr/comm/security_json.cpp | 141 +++ .../security/keymgr/comm/security_utils.cpp | 240 ++++ .../security_aead_aes_hmac_enc_key.cpp} | 54 +- .../encrypt/security_encrypt_decrypt.cpp | 742 ++++++++++++ .../keymgr/encrypt/security_sm2_enc_key.cpp} | 40 +- .../security/keymgr/his/security_his.cpp | 217 ++++ .../security/keymgr/his/security_his_iam.cpp | 298 +++++ .../security/keymgr/his/security_his_kms.cpp | 528 +++++++++ .../security/keymgr/hwc/security_hwc.cpp | 241 ++++ .../security/keymgr/hwc/security_hwc_iam.cpp | 271 +++++ .../security/keymgr/hwc/security_hwc_kms.cpp | 625 ++++++++++ .../keymgr/ktool/security_gs_ktool.cpp} | 325 +++--- .../keymgr/localkms/security_cmkem_comm.cpp | 197 ++++ .../security_cmkem_comm_algorithm.cpp} | 134 +-- .../keymgr/localkms/security_file_enc.cpp | 409 +++++++ .../keymgr/localkms/security_localkms.cpp | 589 ++++++++++ src/include/Makefile | 2 +- src/include/keymgr/comm/security_aksk.h | 57 + .../keymgr/comm/security_encode.h} | 27 +- src/include/keymgr/comm/security_error.h | 111 ++ src/include/keymgr/comm/security_http.h | 83 ++ src/include/keymgr/comm/security_httpscan.h | 97 ++ src/include/keymgr/comm/security_json.h | 37 + src/include/keymgr/comm/security_utils.h | 70 ++ .../keymgr/comm/security_version.h} | 25 +- .../encrypt/security_aead_aes_hamc_enc_key.h} | 17 +- .../encrypt/security_client_logic_enums.h | 125 ++ .../encrypt/security_encrypt_decrypt.h} | 31 +- .../keymgr/encrypt/security_sm2_enc_key.h} | 6 +- src/include/keymgr/his/security_his.h | 49 + src/include/keymgr/his/security_his_iam.h | 52 + src/include/keymgr/his/security_his_kms.h | 59 + src/include/keymgr/hwc/security_hwc.h | 47 + src/include/keymgr/hwc/security_hwc_iam.h | 52 + src/include/keymgr/hwc/security_hwc_kms.h | 55 + .../keymgr/ktool/security_gs_ktool.h} | 27 +- .../keymgr/localkms/security_cmkem_comm.h} | 45 +- .../localkms/security_cmkem_comm_algorithm.h} | 27 +- .../keymgr/localkms/security_file_enc.h | 47 + .../keymgr/localkms/security_localkms.h} | 20 +- src/include/keymgr/security_key_adpt.h | 54 + src/include/keymgr/security_key_mgr.h | 77 ++ src/include/libpq/cl_state.h | 3 + 78 files changed, 7806 insertions(+), 5049 deletions(-) delete mode 100644 src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/Makefile delete mode 100644 src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/cmkem_comm.cpp delete mode 100644 src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/cmkem_comm_http.cpp delete mode 100644 src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/cmkem_comm_http.h delete mode 100644 src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/kms_restful_temp.ini delete mode 100644 src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/reg_cmkem_manager_main.cpp delete mode 100644 src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/reg_hook_frame.cpp delete mode 100644 src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/reg_hook_frame.h delete mode 100644 src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/register_gram_check.cpp delete mode 100644 src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/register_gram_check.h delete mode 100644 src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/register_huawei_kms.cpp delete mode 100644 src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/register_huawei_kms.h delete mode 100644 src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/register_local_kms.cpp delete mode 100755 src/common/interfaces/libpq/client_logic_hooks/encryption_hooks/encrypt_decrypt.cpp create mode 100755 src/gausskernel/security/keymgr/CMakeLists.txt create mode 100644 src/gausskernel/security/keymgr/Makefile create mode 100644 src/gausskernel/security/keymgr/api/security_key_adpt.cpp create mode 100644 src/gausskernel/security/keymgr/comm/security_aksk.cpp create mode 100644 src/gausskernel/security/keymgr/comm/security_encode.cpp create mode 100644 src/gausskernel/security/keymgr/comm/security_error.cpp create mode 100644 src/gausskernel/security/keymgr/comm/security_http.cpp create mode 100644 src/gausskernel/security/keymgr/comm/security_httpscan.cpp create mode 100644 src/gausskernel/security/keymgr/comm/security_json.cpp create mode 100644 src/gausskernel/security/keymgr/comm/security_utils.cpp rename src/{common/interfaces/libpq/client_logic_hooks/encryption_hooks/aead_aes_hamc_enc_key.cpp => gausskernel/security/keymgr/encrypt/security_aead_aes_hmac_enc_key.cpp} (78%) mode change 100755 => 100644 create mode 100644 src/gausskernel/security/keymgr/encrypt/security_encrypt_decrypt.cpp rename src/{common/interfaces/libpq/client_logic_hooks/encryption_hooks/sm2_enc_key.cpp => gausskernel/security/keymgr/encrypt/security_sm2_enc_key.cpp} (90%) create mode 100644 src/gausskernel/security/keymgr/his/security_his.cpp create mode 100644 src/gausskernel/security/keymgr/his/security_his_iam.cpp create mode 100644 src/gausskernel/security/keymgr/his/security_his_kms.cpp create mode 100644 src/gausskernel/security/keymgr/hwc/security_hwc.cpp create mode 100644 src/gausskernel/security/keymgr/hwc/security_hwc_iam.cpp create mode 100644 src/gausskernel/security/keymgr/hwc/security_hwc_kms.cpp rename src/{common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/register_gs_ktool.cpp => gausskernel/security/keymgr/ktool/security_gs_ktool.cpp} (51%) create mode 100644 src/gausskernel/security/keymgr/localkms/security_cmkem_comm.cpp rename src/{common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/cmkem_comm_algorithm.cpp => gausskernel/security/keymgr/localkms/security_cmkem_comm_algorithm.cpp} (66%) create mode 100644 src/gausskernel/security/keymgr/localkms/security_file_enc.cpp create mode 100644 src/gausskernel/security/keymgr/localkms/security_localkms.cpp create mode 100644 src/include/keymgr/comm/security_aksk.h rename src/{common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/reg_cmkem_manager_main.h => include/keymgr/comm/security_encode.h} (55%) create mode 100644 src/include/keymgr/comm/security_error.h create mode 100644 src/include/keymgr/comm/security_http.h create mode 100644 src/include/keymgr/comm/security_httpscan.h create mode 100644 src/include/keymgr/comm/security_json.h create mode 100644 src/include/keymgr/comm/security_utils.h rename src/{common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/cmkem_version_control.h => include/keymgr/comm/security_version.h} (68%) rename src/{common/interfaces/libpq/client_logic_hooks/encryption_hooks/aead_aes_hamc_enc_key.h => include/keymgr/encrypt/security_aead_aes_hamc_enc_key.h} (91%) create mode 100644 src/include/keymgr/encrypt/security_client_logic_enums.h rename src/{common/interfaces/libpq/client_logic_hooks/encryption_hooks/encrypt_decrypt.h => include/keymgr/encrypt/security_encrypt_decrypt.h} (76%) rename src/{common/interfaces/libpq/client_logic_hooks/encryption_hooks/sm2_enc_key.h => include/keymgr/encrypt/security_sm2_enc_key.h} (88%) create mode 100644 src/include/keymgr/his/security_his.h create mode 100644 src/include/keymgr/his/security_his_iam.h create mode 100644 src/include/keymgr/his/security_his_kms.h create mode 100644 src/include/keymgr/hwc/security_hwc.h create mode 100644 src/include/keymgr/hwc/security_hwc_iam.h create mode 100644 src/include/keymgr/hwc/security_hwc_kms.h rename src/{common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/register_gs_ktool.h => include/keymgr/ktool/security_gs_ktool.h} (72%) rename src/{common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/cmkem_comm.h => include/keymgr/localkms/security_cmkem_comm.h} (73%) rename src/{common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/cmkem_comm_algorithm.h => include/keymgr/localkms/security_cmkem_comm_algorithm.h} (78%) create mode 100644 src/include/keymgr/localkms/security_file_enc.h rename src/{common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/register_local_kms.h => include/keymgr/localkms/security_localkms.h} (81%) create mode 100644 src/include/keymgr/security_key_adpt.h create mode 100644 src/include/keymgr/security_key_mgr.h diff --git a/src/common/backend/client_logic/client_logic.cpp b/src/common/backend/client_logic/client_logic.cpp index db6ee6d55..631ba9c24 100755 --- a/src/common/backend/client_logic/client_logic.cpp +++ b/src/common/backend/client_logic/client_logic.cpp @@ -54,7 +54,7 @@ #include "postgres_ext.h" #include "catalog/pg_proc.h" -const size_t ENCRYPTED_VALUE_MIN_LENGTH = 170; +const size_t ENCRYPTED_VALUE_MIN_LENGTH = 116; const size_t ENCRYPTED_VALUE_MAX_LENGTH = 1024; static bool get_cmk_name(const Oid global_key_id, NameData &cmk_name); diff --git a/src/common/backend/parser/gram.y b/src/common/backend/parser/gram.y index c0eab111d..17d8545e6 100644 --- a/src/common/backend/parser/gram.y +++ b/src/common/backend/parser/gram.y @@ -7655,7 +7655,7 @@ master_key_elem: // len is not filled on purpose ?? $$ = (Node*) n; } - | KEY_PATH '=' ColId + | KEY_PATH '=' ColId_or_Sconst { ClientLogicGlobalParam *n = makeNode (ClientLogicGlobalParam); n->key = ClientLogicGlobalProperty::CMK_KEY_PATH; diff --git a/src/common/interfaces/libpq/CMakeLists.txt b/src/common/interfaces/libpq/CMakeLists.txt index 420c94075..b899d1bde 100755 --- a/src/common/interfaces/libpq/CMakeLists.txt +++ b/src/common/interfaces/libpq/CMakeLists.txt @@ -165,9 +165,9 @@ SET(TGT_pq_ce_INC ${CMAKE_CURRENT_SOURCE_DIR}/client_logic_cache ${CMAKE_CURRENT_SOURCE_DIR}/client_logic_processor ${CMAKE_CURRENT_SOURCE_DIR}/client_logic_fmt - ${CMAKE_CURRENT_SOURCE_DIR}/client_logic_hooks/cmk_entity_manager_hooks ${CMAKE_CURRENT_SOURCE_DIR}/client_logic_hooks/encryption_hooks ${CMAKE_CURRENT_SOURCE_DIR}/client_logic_data_fetcher + ${PROJECT_SRC_DIR}/gausskernel/security/keymgr/include ${JAVA_HOME}/include ${JAVA_HOME}/include/linux ) @@ -209,23 +209,6 @@ set(libpq_ce_COMPILE_OPTIONS ${pq_ce_COMPILE_OPTIONS}) list(REMOVE_ITEM libpq_ce_COMPILE_OPTIONS -fstack-protector -pthread) add_static_objtarget(libpq_ce TGT_libpq_ce_SRC TGT_pq_ce_INC "${libpq_ce_DEF_OPTIONS}" "${libpq_ce_COMPILE_OPTIONS}" "") -AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR}/client_logic_hooks/cmk_entity_manager_hooks TGT_cmk_entity_manager_hooks_SRC) -if(NOT "${ENABLE_UT}" STREQUAL "ON") - if(NOT "${ENABLE_MULTIPLE_NODES}_${ENABLE_PRIVATEGAUSS}" STREQUAL "OFF_OFF") - if(NOT "${ENABLE_LITE_MODE}" STREQUAL "ON") - list(REMOVE_ITEM TGT_cmk_entity_manager_hooks_SRC ${CMAKE_CURRENT_SOURCE_DIR}/client_logic_hooks/cmk_entity_manager_hooks/register_local_kms.cpp) - else() - list(REMOVE_ITEM TGT_cmk_entity_manager_hooks_SRC ${CMAKE_CURRENT_SOURCE_DIR}/client_logic_hooks/cmk_entity_manager_hooks/register_huawei_kms.cpp) - list(REMOVE_ITEM TGT_cmk_entity_manager_hooks_SRC ${CMAKE_CURRENT_SOURCE_DIR}/client_logic_hooks/cmk_entity_manager_hooks/register_gs_ktool.cpp) - endif() - else() - list(REMOVE_ITEM TGT_cmk_entity_manager_hooks_SRC ${CMAKE_CURRENT_SOURCE_DIR}/client_logic_hooks/cmk_entity_manager_hooks/register_huawei_kms.cpp) - list(REMOVE_ITEM TGT_cmk_entity_manager_hooks_SRC ${CMAKE_CURRENT_SOURCE_DIR}/client_logic_hooks/cmk_entity_manager_hooks/register_gs_ktool.cpp) - endif() -endif() -set(cmk_entity_manager_hooks_DEF_OPTIONS ${MACRO_OPTIONS} -DFRONTEND -DFRONTEND_PARSER -DUNSAFE_STAT_OK -DHAVE_CE -DWORDS_BIGENDIAN -DSO_MAJOR_VERSION=5) -add_static_objtarget(cmk_entity_manager_hooks TGT_cmk_entity_manager_hooks_SRC TGT_pq_ce_INC "${cmk_entity_manager_hooks_DEF_OPTIONS}" "${pq_ce_COMPILE_OPTIONS}" "") - AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR}/client_logic_hooks/encryption_hooks TGT_encryption_hooks_SRC) set(encryption_hooks_DEF_OPTIONS ${MACRO_OPTIONS} -DFRONTEND -DFRONTEND_PARSER -DHAVE_CE) list(REMOVE_ITEM encryption_hooks_DEF_OPTIONS -D_GNU_SOURCE) @@ -289,18 +272,18 @@ add_static_objtarget(frontend_parser TGT_frontend_parser_SRC TGT_pq_ce_INC "${fr set(pq_ce_LINK_OPTIONS ${LIB_LINK_OPTIONS}) add_shared_libtarget(pq_ce TGT_pq_ce_SRC TGT_pq_ce_INC "${pq_ce_DEF_OPTIONS}" "${pq_ce_COMPILE_OPTIONS}" "${pq_ce_LINK_OPTIONS}") -set(pq_ce_LINK_LIBS libpq_ce cmk_entity_manager_hooks encryption_hooks client_logic_common client_logic_expressions client_logic_cache client_logic_processor client_logic_fmt client_logic_hooks client_logic_data_fetcher frontend_parser) +set(pq_ce_LINK_LIBS libpq_ce encryption_hooks client_logic_common client_logic_expressions client_logic_cache client_logic_processor client_logic_fmt client_logic_hooks client_logic_data_fetcher frontend_parser) if(NOT "${ENABLE_MULTIPLE_NODES}_${ENABLE_PRIVATEGAUSS}" STREQUAL "OFF_OFF") if(NOT "${ENABLE_LITE_MODE}" STREQUAL "ON") set(pq_ce_LINK_LIBS ${pq_ce_LINK_LIBS} -lgs_ktool -lkmc) endif() endif() if(NOT "${ENABLE_LITE_MODE}" STREQUAL "ON") - target_link_libraries(pq_ce PRIVATE ${pq_ce_LINK_LIBS} -lcurl -lcjson -lssl -lcrypto -l${SECURE_C_CHECK} -lpthread -lgssapi_krb5_gauss -lgssrpc_gauss -lkrb5_gauss -lkrb5support_gauss -lk5crypto_gauss -lcom_err_gauss) + target_link_libraries(pq_ce PRIVATE ${pq_ce_LINK_LIBS} libkeymgr.a -lcurl -lcjson -lssl -lcrypto -l${SECURE_C_CHECK} -lpthread -lgssapi_krb5_gauss -lgssrpc_gauss -lkrb5_gauss -lkrb5support_gauss -lk5crypto_gauss -lcom_err_gauss) else() - target_link_libraries(pq_ce PRIVATE ${pq_ce_LINK_LIBS} -lcurl -lcjson -lssl -lcrypto -l${SECURE_C_CHECK} -lpthread) + target_link_libraries(pq_ce PRIVATE ${pq_ce_LINK_LIBS} libkeymgr.a -lcurl -lcjson -lssl -lcrypto -l${SECURE_C_CHECK} -lpthread) endif() -add_dependencies(pq_ce libpq_ce cmk_entity_manager_hooks encryption_hooks client_logic_common client_logic_expressions client_logic_cache client_logic_processor client_logic_fmt client_logic_hooks client_logic_data_fetcher frontend_parser) +add_dependencies(pq_ce libpq_ce encryption_hooks client_logic_common client_logic_expressions client_logic_cache client_logic_processor client_logic_fmt client_logic_hooks client_logic_data_fetcher frontend_parser) target_link_directories(pq_ce PUBLIC ${SECURE_LIB_PATH} ${KMC_LIB_PATH} @@ -313,6 +296,7 @@ target_link_directories(pq_ce PUBLIC if(NOT "${ENABLE_MULTIPLE_NODES}_${ENABLE_PRIVATEGAUSS}" STREQUAL "OFF_OFF") if(NOT "${ENABLE_LITE_MODE}" STREQUAL "ON") add_dependencies(pq_ce gs_ktool) + add_dependencies(pq_ce keymgr) endif() endif() SET_TARGET_PROPERTIES(pq_ce PROPERTIES VERSION 5.5) diff --git a/src/common/interfaces/libpq/Makefile b/src/common/interfaces/libpq/Makefile index 2ed81c9fa..b58cd71ec 100644 --- a/src/common/interfaces/libpq/Makefile +++ b/src/common/interfaces/libpq/Makefile @@ -123,27 +123,11 @@ SHLIB_EXPORTS = exports.txt ifeq "$(ENABLE_CE)" "1" SUBDIRS = frontend_parser client_logic_common client_logic_expressions client_logic_cache client_logic_processor client_logic_fmt client_logic_hooks client_logic_data_fetcher encryption_hooks_files := $(shell find client_logic_hooks/encryption_hooks -name '*.cpp' | sort) -cmk_entity_manager_hooks_files := $(shell find client_logic_hooks/cmk_entity_manager_hooks -maxdepth 1 -name '*.cpp' | sort) - +key_files := $(shell find $(top_builddir)/src/gausskernel/security/keymgr -name '*.cpp' | sort) OBJS += $(encryption_hooks_files:.cpp=.o) -OBJS += $(cmk_entity_manager_hooks_files:.cpp=.o) +OBJS += $(key_files:.cpp=.o) endif -ifneq ($(enable_ut), yes) -ifneq ($(enable_multiple_nodes)_$(enable_privategauss), no_no) -ifneq ($(enable_lite_mode), yes) - OBJS := $(filter-out client_logic_hooks/cmk_entity_manager_hooks/register_local_kms.o, $(OBJS)) -else - OBJS := $(filter-out client_logic_hooks/cmk_entity_manager_hooks/register_huawei_kms.o, $(OBJS)) - OBJS := $(filter-out client_logic_hooks/cmk_entity_manager_hooks/register_gs_ktool.o, $(OBJS)) -endif -else -OBJS := $(filter-out client_logic_hooks/cmk_entity_manager_hooks/register_huawei_kms.o, $(OBJS)) -OBJS := $(filter-out client_logic_hooks/cmk_entity_manager_hooks/register_gs_ktool.o, $(OBJS)) -endif -endif - - files:= $(foreach DIR,$(SUBDIRS), $(wildcard $(DIR)/*.cpp) ) ifneq "$(ENABLE_CE)" "1" files += $(srcdir)/frontend_parser/fe-wchar.o $(srcdir)/frontend_parser/frontend_mbutils.o $(srcdir)/frontend_parser/wstrncmp.cpp @@ -177,7 +161,6 @@ libpq_ce: endif - frontend_parser/gram.hpp: $(MAKE) -C frontend_parser gram.hpp frontend_parser/parser.o frontend_parser/kwlookup.o:frontend_parser/gram.hpp diff --git a/src/common/interfaces/libpq/cl_state.cpp b/src/common/interfaces/libpq/cl_state.cpp index 5f6e6409b..2231756ca 100644 --- a/src/common/interfaces/libpq/cl_state.cpp +++ b/src/common/interfaces/libpq/cl_state.cpp @@ -34,6 +34,7 @@ #include "client_logic_cache/cached_proc.h" #include "client_logic_cache/dataTypes.def" #include "client_logic_data_fetcher/data_fetcher_manager.h" +#include "keymgr/security_key_adpt.h" PGClientLogic::PGClientLogic(PGconn *conn, JNIEnv *java_env, jobject jdbc_handle) : m_conn(conn), @@ -68,6 +69,7 @@ PGClientLogic::PGClientLogic(PGconn *conn, JNIEnv *java_env, jobject jdbc_handle m_data_fetcher_manager = new DataFetcherManager(conn, java_env, jdbc_handle); m_parser_context.eaten_begin = false; m_parser_context.eaten_declare = false; + m_key_adpt = key_adpt_new(); } PGClientLogic::~PGClientLogic() @@ -105,6 +107,8 @@ PGClientLogic::~PGClientLogic() } m_cached_column_manager->clear(); delete m_cached_column_manager; + + key_adpt_free((KeyAdpt *)m_key_adpt); } void PGClientLogic::clear_functions_list() diff --git a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/Makefile b/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/Makefile deleted file mode 100644 index a99cd312a..000000000 --- a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/Makefile +++ /dev/null @@ -1,77 +0,0 @@ -#------------------------------------------------------------------------- -# -# Copyright (c) 2020 Huawei Technologies Co.,Ltd. -# -# src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/ -# -#------------------------------------------------------------------------- - -subdir = src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/ -top_builddir = ../../../../../../ - -include $(top_builddir)/src/Makefile.global - -override CPPFLAGS += -I. -I$(top_builddir)/src/include -I../encryption_hooks -override LDFLAGS += -L. -L$(LIBOPENSSL_LIB_PATH) -override LDLIBS += -l$(SECURE_C_CHECK) -lssl -lcrypto -CMKEM_OBJS = common.o cmkem_comm_algorithm.o reg_hook_frame.o - -override CPPFLAGS += -ftrapv -fstack-protector-strong -override LDFLAGS += -ldl -lrt -Wl,-z,relro,-z,now - -# add new cmk managerment tool/components/services here # -# if you need to copy .so/.a to install lib dir, please use '$(INSTALL_DATA) ...' # - -# (1) gs_ktool -override CPPFLAGS += -I$(top_builddir)/src/include/gs_ktool -I$(LIBKMC_INCLUDE_PATH) -override LDFLAGS += -L$(LIBKMC_LIB_PATH) -override LDLIBS += -lkmc -lgs_ktool -CMKEM_OBJS += register_gs_ktool.o - -# (2) localkms -override CPPFLAGS += -I$(CJSON_INCLUDE_PATH) -I$(LIBCURL_INCLUDE_PATH) -override LDFLAGS += -L$(CJSON_LIB_PATH) -L$(LIBCURL_LIB_PATH) -override LDLIBS += -lcjson -lcurl -CMKEM_OBJS += register_local_kms.o - -# (3) huawei_kms -override CPPFLAGS += -override LDFLAGS += -override LDLIBS += -CMKEM_OBJS += register_huawei_kms.o - -# (4) remain to add new tool/components/services -# ... - -# (x) grammar_check -CMKEM_OBJS += register_gram_check.o - -### end ### - - -CMKEM_OBJS += reg_cmkem_manager_main.o -override CPPFLAGS := $(filter-out -fPIE, $(CPPFLAGS)) -fPIC -shared - -BIN_OBJS = $(CMKEM_OBJS) test.o - -all: libcmk_entity_manager_hooks.so - -libcmk_entity_manager_hooks.so: $(CMKEM_OBJS) - echo $(CMKEM_OBJS) - $(CC) -fPIC -shared $(CPPFLAGS) $^ $(LDFLAGS) $(LDLIBS) -o $@ - -install: cmktest installdirs - -cmktest: $(BIN_OBJS) - echo $(LDLIBS) - echo $(BIN_OBJS) - #gcc $(BIN_OBJS) $(CPPFLAGS) $(LDFLAGS) -L. -lcmk_entity_manager_hooks -lgs_ktool -lcjson -lcurl -o $@ - $(CC) -fPIE -pie $(CPPLAGS) $^ $(LDFLAGS) $(LDLIBS) -lcmk_entity_manager_hooks -o $@ - -installdirs: cmktest - $(INSTALL_DATA) libgs_ktool.so$(X) '$(DESTDIR)$(bindir)/../lib/libgs_ktool.so$(X)' - $(INSTALL_DATA) libcmk_entity_manager_hooks.so$(X) '$(DESTDIR)$(bindir)/../lib/libcmk_entity_manager_hooks.so$(X)' - $(INSTALL_PROGRAM) cmktest$(X) '$(DESTDIR)$(bindir)/cmktest$(X)' - -clean distclean maintainer-clean: - rm -f ./*.o libcmk_entity_manager_hooks.so cmktest core.* diff --git a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/cmkem_comm.cpp b/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/cmkem_comm.cpp deleted file mode 100644 index 2e1694fac..000000000 --- a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/cmkem_comm.cpp +++ /dev/null @@ -1,720 +0,0 @@ -/* - * Copyright (c) 2020 Huawei Technologies Co.,Ltd. - * - * openGauss is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * - * http://license.coscl.org.cn/MulanPSL2 - * - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - * ------------------------------------------------------------------------- - * - * cmkem_comm.cpp - * some common functions, include: - * 1. error code and error process - * 2. string process - * 3. format and conversion - * - * IDENTIFICATION - * src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/cmkem_comm.cpp - * - * ------------------------------------------------------------------------- - */ - -#include "cmkem_comm.h" - -#include -#include -#include -#include -#include -#include -#include -#include "securec.h" -#include "securec_check.h" - -static char cmkem_errmsg_buf[MAX_CMKEM_ERRMSG_BUF_SIZE] = {0}; - -/* - * in some cases, where there is an error, we will report the error in detail. - * if not, we will use simple and general error information according to the error code - */ -static CmkemErrMsg simple_errmsg_list[] = { - {CMKEM_SUCCEED, "no error message, the result is 'SUCCEED'."}, - {CMKEM_UNKNOWN_ERR, "unknown error."}, - {CMKEM_CHECK_STORE_ERR, "failed to check the value of arg: 'KEY_STORE'."}, - {CMKEM_CHECK_ALGO_ERR, "failed to check the value of arg: 'ALGORITHM'."}, - {CMKEM_CHECK_CMK_ID_ERR, "failed to check the value of arg: 'KEY_PATH'."}, - {CMKEM_CHECK_IDENTITY_ERR, "faiel to check the usability of cmk entity."}, - {CMKEM_CHECK_INPUT_AUTH_ERR, "the authentication parameters are illegal."}, - {CMKEM_CHECK_PROJECT_NAME_ERR, "failed to check HuaWei KMS project name."}, - {CMKEM_GET_TOKEN_ERR, "failed to get token."}, - {CMKEM_WRITE_TO_BIO_ERR, "failed to wirite key to BIO buffer."}, - {CMKEM_READ_FROM_BIO_ERR, "failed to read key from BIO buffer."}, - {CMKEM_GET_RAND_NUM_ERR, "failed to generate a random num."}, - {CMKEM_DERIVED_KEY_ERR, "failed to derived key."}, - {CMKEM_GEN_RSA_KEY_ERR, "failed to create rsa key pair."}, - {CMKEM_GEN_SM2_KEY_ERR, "failed to create sm2 key pair."}, - {CMKEM_RSA_DECRYPT_ERR, "failed to decrypt cipher with RSA algorithm."}, - {CMKEM_RSA_ENCRYPT_ERR, "failed to encrypt plain with RSA algorithm."}, - {CMKEM_CHECK_HASH_ERR, "failed check the hash of data."}, - {CMKEM_EVP_ERR, "openssl evp error."}, - {CMKEM_SM2_ENC_ERR, "failed to encrypt plain with SM2 algorithm."}, - {CMKEM_SM2_DEC_ERR, "failed to decrypt cipher with SM2 algorithm."}, - {CMKEM_REG_CMK_MANAGER_ERR, "failed to register cmk entity manager."}, - {CMKEM_REGISTRE_FUNC_ERR, "failed to register hook function of cmk manager."}, - {CMKEM_CREATE_CMK_ERR, "failed to create client master key object."}, - {CMKEM_ENCRYPT_CEK_ERR, "failed to encrypt column encryption key plain with client master key."}, - {CMKEM_DECRYPT_CEK_ERR, "failed to decrypt column encryption key cipher with client master key."}, - {CMKEM_DROP_CMK_ERR, "failed to drop client master key object."}, - {CMKEM_GET_ENV_VAL_ERR, "failed to get environment value from system."}, - {CMKEM_CHECK_ENV_VAL_ERR, "the environment value is illegal."}, - {CMKEM_SET_ENV_VALUE_ERR, "failed to set global environment value."}, - {CMKEM_GS_KTOOL_ERR, "gs_ktool error."}, - {CMKEM_IAM_SERVER_ERR, "Huawei IAM server error."}, - {CMKEM_CHECK_CHACHE_ID_ERR, "the client cache id is invalid."}, - {CMKEM_CACHE_IS_EMPTY, "the specific cache block is empty."}, - {CMKEM_TOKEN_EXPIRED_ERR, "the token has expired."}, - {CMKEM_KMS_SERVER_ERR, "Huawei KMS server error."}, - {CMKEM_ENC_CMK_ERR, "failed to encrypt cmk entity plain."}, - {CMKEM_DEC_CMK_ERR, "failed to decrypt cmk entity cipher."}, - {CMKEM_CREATE_FILE_ERR, "failed to create file."}, - {CMKEM_FIND_FILE_ERR, "failed to find file."}, - {CMKEM_OPEN_FILE_ERR, "failed to open file."}, - {CMKEM_READ_FILE_ERR, "failed to read file."}, - {CMKEM_READ_FILE_STATUS_ERR, "failed to read file status."}, - {CMKEM_WRITE_FILE_ERR, "failed to write to file."}, - {CMKEM_REMOVE_FILE_ERR, "failed to remove file."}, - {CMKEM_MALLOC_MEM_ERR, "failed to malloc memory from heap."}, - {CMKEM_CHECK_BUF_LEN_ERR, "the buffer size is too short."}, - {CMKEM_CJSON_PARSE_ERR, "failed to parser string with 'CJSON'."}, - {CMKEM_FIND_CSJON_ERR, "failed to find node from cjson tree."}, - {CMKEM_SET_CJSON_VALUE_ERR, "failed to set the value of cjson tree node."}, - {CMKEM_CURL_INIT_ERR, "failed to init curl."}, - {CMKEM_CURL_ERR, "curl error."}, -}; - -/* - * write error message to err_msg buffer, if need error message, can chose: - * 1. cmkem_list_print error message. - * 2. copy/dump error message to new err_msg buffer. - */ -void write_cmkem_errmsg(const char *errmsg) -{ - errno_t rc = 0; - - if (cmkem_errmsg_buf == NULL) { - return; - } - - rc = memset_s(cmkem_errmsg_buf, MAX_CMKEM_ERRMSG_BUF_SIZE, 0, sizeof(cmkem_errmsg_buf)); - securec_check_c(rc, "", ""); - - rc = sprintf_s(cmkem_errmsg_buf, MAX_CMKEM_ERRMSG_BUF_SIZE, "%s", errmsg); - securec_check_ss_c(rc, "", ""); -} - -void print_cmkem_errmsg_buf() -{ - if (cmkem_errmsg_buf == NULL) { - printf("ERROR: error message buffer is 'NULL'."); - return; - } - - if (strlen(cmkem_errmsg_buf) > 0) { - printf("ERROR: %s\n", cmkem_errmsg_buf); - } else { - printf("NOTICE: error message buffer is empty.\n"); - } -} - -void dump_cmkem_errmsg_buf(char *dest_buf, size_t dest_buf_size) -{ - const char *empty_hint = "error message buffer is empty."; - const char *buf_too_short_hint = "failed to dump error message, your buffer is too short."; - errno_t rc = 0; - - if (cmkem_errmsg_buf == NULL) { - return; - } - - if (strlen(cmkem_errmsg_buf) == 0) { - if (strlen(empty_hint) < dest_buf_size) { - rc = strcpy_s(dest_buf, dest_buf_size, empty_hint); - securec_check_c(rc, "", ""); - } - return; - } - - if (strlen(cmkem_errmsg_buf) > dest_buf_size) { - if (strlen(buf_too_short_hint) < dest_buf_size) { - rc = strcpy_s(dest_buf, dest_buf_size, buf_too_short_hint); - securec_check_c(rc, "", ""); - } - return; - } - - rc = strcpy_s(dest_buf, dest_buf_size, cmkem_errmsg_buf); - securec_check_c(rc, "", ""); - - return; -} - -const char *get_cmkem_errmsg(CmkemErrCode err_code) -{ - if (cmkem_errmsg_buf == NULL) { - return NULL; - } - - if (strlen(cmkem_errmsg_buf) == 0) { - return simple_errmsg_list[err_code].err_msg; - } - - return (const char *)cmkem_errmsg_buf; -} - -void erase_data(void *data, size_t data_len) -{ - errno_t rc = 0; - - if (data == NULL) { - return; - } - - rc = memset_s(data, data_len, 0, data_len); - securec_check_c(rc, "", ""); -} - -CmkemStr *malloc_cmkem_str(size_t str_buf_len) -{ - if (str_buf_len == 0) { - return NULL; - } - - CmkemStr *advstr = (CmkemStr *)malloc(sizeof(CmkemStr)); - if (advstr == NULL) { - return NULL; - } - - advstr->str_val = (char *)malloc(str_buf_len); - if (advstr->str_val == NULL) { - cmkem_free(advstr); - return NULL; - } - - advstr->str_len = 0; - return advstr; -} - -void free_cmkem_str(CmkemStr *advstr_ptr) -{ - if (advstr_ptr != NULL) { - cmkem_free(advstr_ptr->str_val); - cmkem_free(advstr_ptr); - } - - advstr_ptr = NULL; -} - -CmkemUStr *malloc_cmkem_ustr(size_t ust_buf_len) -{ - if (ust_buf_len == 0) { - return NULL; - } - - CmkemUStr *key_str = (CmkemUStr *)malloc(sizeof(CmkemUStr)); - if (key_str == NULL) { - return NULL; - } - - key_str->ustr_val = (unsigned char *)malloc(ust_buf_len); - if (key_str->ustr_val == NULL) { - cmkem_free(key_str); - return NULL; - } - - key_str->ustr_len = 0; - return key_str; -} - -void free_cmkem_ustr(CmkemUStr *cmkem_ustr) -{ - if (cmkem_ustr != NULL) { - cmkem_free(cmkem_ustr->ustr_val); - cmkem_free(cmkem_ustr); - } - - cmkem_ustr = NULL; -} - -void free_cmkem_ustr_with_erase(CmkemUStr *cmkem_ustr) -{ - if (cmkem_ustr != NULL) { - erase_data(cmkem_ustr->ustr_val, cmkem_ustr->ustr_len); - cmkem_free(cmkem_ustr->ustr_val); - cmkem_free(cmkem_ustr); - } - - cmkem_ustr = NULL; -} - -CmkemStrList *malloc_cmkem_list(void) -{ - CmkemStrList *new_list = NULL; - - new_list = (CmkemStrList *)malloc(sizeof(CmkemStrList)); - if (new_list == NULL) { - return NULL; - } - - new_list->node_cnt = 0; - new_list->first_node = NULL; - return new_list; -} - -void free_cmkem_list(CmkemStrList *list) -{ - CmkemStrNode *cur_node = NULL; - CmkemStrNode *to_free = NULL; - - if (list == NULL) { - return; - } - - cur_node = list->first_node; - while (cur_node != NULL) { - to_free = cur_node; - cur_node = cur_node->next; - cmkem_free(to_free->str_val); - cmkem_free(to_free); - } - - cmkem_free(list); -} - -void free_cmkem_list_with_skip(CmkemStrList *list, int list_pos) -{ - CmkemStrNode *cur_node = NULL; - CmkemStrNode *to_free = NULL; - - if (list_pos == -1) { - list_pos = list->node_cnt - 1; - } - - cur_node = list->first_node; - for (int i = 0; i < (int)cmkem_list_len(list); i++) { - to_free = cur_node; - cur_node = cur_node->next; - - if (i == list_pos) { - cmkem_free(to_free); - } else { - cmkem_free(to_free->str_val); - cmkem_free(to_free); - } - } - - cmkem_free(list); -} - -size_t cmkem_list_len(CmkemStrList *list) -{ - return list->node_cnt; -} - -void cmkem_list_size(CmkemStrList *list, size_t *list_size) -{ - for (size_t i = 0; i < cmkem_list_len(list); i++) { - *list_size += strlen(cmkem_list_val(list, i)); - } -} - -char *cmkem_list_val(CmkemStrList *list, int list_pos) -{ - CmkemStrNode *target_node = NULL; - - if (list_pos == -1) { - list_pos = list->node_cnt - 1; - } - - if (list_pos < 0 || list_pos >= list->node_cnt) { - return NULL; - } - - target_node = list->first_node; - for (int i = 0; i < list_pos; i++) { - target_node = target_node->next; - } - - return target_node->str_val; -} - -int cmkem_list_pos(CmkemStrList *list, const char *str_val) -{ - CmkemStrNode *target_node = NULL; - - if (list->node_cnt < 1) { - return -1; /* -1 means : cannot find str_val in list */ - } - - target_node = list->first_node; - for (int i = 0; i < list->node_cnt; i++) { - if (strcmp(target_node->str_val, str_val) == 0) { - return i; - } - target_node = target_node->next; - } - - return -1; -} - -void cmkem_list_print(CmkemStrList *list) -{ - CmkemStrNode *cur_node = NULL; - - printf("(%d)[", list->node_cnt); - - cur_node = list->first_node; - for (int i = 0; i < list->node_cnt; i++) { - if (i != list->node_cnt - 1) { - printf("'%s', ", cur_node->str_val); - } else { - /* cmkem_list_print the last node */ - printf("'%s'", cur_node->str_val); - } - cur_node = cur_node->next; - } - - printf("]\n"); -} - -/* - * malloc a new_node, then: - * the last_node -> next = new_node - */ -void cmkem_list_append(CmkemStrList *list, const char *str_val) -{ - CmkemStrNode *last_node = NULL; - CmkemStrNode *new_node = NULL; - char *new_node_str_val = NULL; - errno_t rc = 0; - - if (str_val == NULL) { - return; - } - - new_node_str_val = (char *)malloc(strlen(str_val) + 1); - if (new_node_str_val == NULL) { - return; - } - - new_node = (CmkemStrNode *)malloc(sizeof(CmkemStrNode)); - if (new_node == NULL) { - cmkem_free(new_node_str_val); - return; - } - rc = strcpy_s(new_node_str_val, strlen(str_val) + 1, str_val); - securec_check_c(rc, "", ""); - - new_node->str_val = new_node_str_val; - new_node->next = NULL; - - if (list->first_node == NULL) { - list->first_node = new_node; - } else { - last_node = list->first_node; - for (int i = 0; i < (list->node_cnt - 1); i++) { - last_node = last_node->next; - } - last_node->next = new_node; - } - - list->node_cnt++; -} - -void cmkem_list_insert(CmkemStrList *list, const char *str_val, int list_pos) -{ - CmkemStrNode *new_node = NULL; - CmkemStrNode *pre_new_node = NULL; - char *new_node_str_val = NULL; - errno_t rc = 0; - - if (list_pos < 0 || list_pos >= list->node_cnt) { - return; - } - - new_node_str_val = (char *)malloc(strlen(str_val) + 1); - if (new_node_str_val == NULL) { - return; - } - - new_node = (CmkemStrNode *)malloc(sizeof(CmkemStrNode)); - if (new_node == NULL) { - cmkem_free(new_node_str_val); - return; - } - - rc = strcpy_s(new_node_str_val, strlen(str_val) + 1, str_val); - securec_check_c(rc, "", ""); - new_node->str_val = new_node_str_val; - - if (list_pos == 0) { - new_node->next = list->first_node; - list->first_node = new_node; - } else { - pre_new_node = list->first_node; - for (int i = 0; i < (list_pos - 1); i++) { - pre_new_node = pre_new_node->next; - } - new_node->next = pre_new_node->next; - pre_new_node->next = new_node; - } - - list->node_cnt++; -} - -void cmkem_list_pop(CmkemStrList *list) -{ - CmkemStrNode *last_node = NULL; - CmkemStrNode *pre_last_node = NULL; - - if (list->first_node == 0) { - return; - } else if (list->node_cnt == 1) { - last_node = list->first_node; - list->first_node = NULL; - } else { - pre_last_node = list->first_node; - for (int i = 0; i < (list->node_cnt - 2); i++) { - pre_last_node = pre_last_node->next; - } - last_node = pre_last_node->next; - pre_last_node->next = NULL; - } - - cmkem_free(last_node->str_val); - cmkem_free(last_node); - list->node_cnt--; -} - -void cmkem_list_del(CmkemStrList *list, int list_pos) -{ - CmkemStrNode *target_node = NULL; - CmkemStrNode *pre_target_node = NULL; - - if (list_pos == -1) { - list_pos = list->node_cnt - 1; - } - - if (list_pos == 0) { - target_node = list->first_node; - list->first_node = list->first_node->next; - } else if (list_pos > 0 && list_pos < list->node_cnt) { - pre_target_node = list->first_node; - for (int i = 0; i < list_pos - 1; i++) { - pre_target_node = pre_target_node->next; - } - target_node = pre_target_node->next; - pre_target_node->next = target_node->next; - } else { - return; - } - - cmkem_free(target_node->str_val); - cmkem_free(target_node); - list->node_cnt--; -} - -void cmkem_list_cat(CmkemStrList *base_list, CmkemStrList *extra_list) -{ - CmkemStrNode *last_node = NULL; - - if (base_list->node_cnt == 0) { - base_list->first_node = extra_list->first_node; - base_list->node_cnt += extra_list->node_cnt; - return; - } else { - last_node = base_list->first_node; - for (int i = 0; i < (base_list->node_cnt - 1); i++) { - last_node = last_node->next; - } - last_node->next = extra_list->first_node; - base_list->node_cnt += extra_list->node_cnt; - } -} - -char *cmkem_list_merge(CmkemStrList* list) -{ - size_t all_strlen = 1; /* for EOF */ - char *merged_str = NULL; - errno_t rc = 0; - - for (size_t i = 0; i < cmkem_list_len(list); i++) { - all_strlen += strlen(cmkem_list_val(list, i)); - } - - merged_str = (char *)malloc(all_strlen); - if (merged_str == NULL) { - return NULL; - } - - for (size_t i = 0; i < cmkem_list_len(list); i++) { - if (i == 0) { - rc = strcpy_s(merged_str, all_strlen, cmkem_list_val(list, i)); - securec_check_c(rc, "", ""); - } else { - rc = strcat_s(merged_str, all_strlen, cmkem_list_val(list, i)); - securec_check_ss_c(rc, "", ""); - } - } - - return merged_str; -} - -CmkemStrList *cmkem_split(const char *str, char split_char) -{ - CmkemStrList *substr_list = NULL; - size_t str_start = 0; - errno_t rc = 0; - - substr_list = malloc_cmkem_list(); - if (substr_list == NULL) { - return NULL; - } - - for (size_t i = 0; i < strlen(str); i++) { - if (str[i] == split_char || i == strlen(str) - 1) { - if (i == strlen(str) - 1 && str[i] != split_char) { - cmkem_list_append(substr_list, str + str_start); - } else { - char cur_substr[i - str_start + 1] = {0}; - rc = strncpy_s(cur_substr, i - str_start + 1, str + str_start, i - str_start); - securec_check_c(rc, "", ""); - - cur_substr[i - str_start] = '\0'; - str_start = i + 1; - - cmkem_list_append(substr_list, cur_substr); - } - } - } - - if (cmkem_list_len(substr_list) == 0) { - cmkem_list_free(substr_list); - return NULL; - } - - return substr_list; -} - -bool is_str_empty(const char *str) -{ - return (str == NULL || strlen(str) == 0) ? true : false; -} - -CmkemStr *conv_str_to_cmkem_str(const char *str) -{ - errno_t rc = 0; - - CmkemStr *advstr = malloc_cmkem_str(strlen(str) + 1); - if (advstr == NULL) { - return NULL; - } - - rc = strcpy_s(advstr->str_val, strlen(str) + 1, str); - securec_check_c(rc, "", ""); - advstr->str_len = strlen(str); - - return advstr; -} - -CmkemUStr *conv_str_to_cmkem_ustr(const unsigned char *ustr, size_t ustr_len) -{ - CmkemUStr *cmkem_ustr = malloc_cmkem_ustr(ustr_len + 1); - if (cmkem_ustr == NULL) { - return NULL; - } - - for (size_t i = 0; i < ustr_len; i++) { - cmkem_ustr->ustr_val[i] = ustr[i]; - } - cmkem_ustr->ustr_val[ustr_len] = '\0'; - cmkem_ustr->ustr_len = ustr_len; - - return cmkem_ustr; -} - -void push_char(CmkemStr *advstr_ptr, char chr) -{ - advstr_ptr->str_val[advstr_ptr->str_len] = chr; - advstr_ptr->str_len += 1; - advstr_ptr->str_val[advstr_ptr->str_len] = '\0'; -} - -void itoa(int num, char *str_buf, size_t buf_len) -{ - errno_t rc = 0; - - rc = sprintf_s(str_buf, buf_len, "%d", num); - securec_check_ss_c(rc, "", ""); -} - -int hex_atoi(char hex) -{ - if (hex >= '0' && hex <= '9') { - return hex - '0'; - } else if (hex >= 'a' && hex <= 'f') { - return hex - 'a' + 0xa; - } else if (hex >= 'A' && hex <= 'F') { - return hex - 'A' + 0xa; - } - - return 0; -} - -CmkemStr *ustr_to_hex(CmkemUStr *ustr) -{ - const char *hex_tbl = "0123456789abcdef"; - size_t pos = 0; - - CmkemStr *hex = malloc_cmkem_str(ustr->ustr_len * 2 + 1); /* e.g ustr = "b", hex = "62" + '\0' */ - if (hex == NULL) { - return NULL; - } - - for (; pos < ustr->ustr_len; pos++) { - hex->str_val[2 * pos] = hex_tbl[(ustr->ustr_val[pos] >> 4) & 0xf]; - hex->str_val[2 * pos + 1] = hex_tbl[((ustr->ustr_val[pos] << 4) >> 4) & 0xf]; - } - hex->str_val[2 * pos] = '\0'; - hex->str_len = 2 * pos; - - return hex; -} - -CmkemUStr *hex_to_ustr(CmkemStr *hex) -{ - size_t pos = 0; - unsigned int low = 0; - unsigned int high = 0; - - CmkemUStr *ustr = malloc_cmkem_ustr((int) hex->str_len / 2 + 1); /* e.g hex = "62", ustr = 'b' + '\0' */ - if (ustr == NULL) { - return NULL; - } - - for (; pos < hex->str_len; pos += 2) { - high = (unsigned int)hex_atoi(hex->str_val[pos]); - low = (unsigned int)hex_atoi(hex->str_val[pos + 1]); - - ustr->ustr_val[(int) pos / 2] = ((high << 4) & 0xff) ^ (low & 0xff); - } - ustr->ustr_val[(int) pos / 2] = '\0'; - ustr->ustr_len = (size_t) pos / 2; - - return ustr; -} diff --git a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/cmkem_comm_http.cpp b/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/cmkem_comm_http.cpp deleted file mode 100644 index 8fec53e91..000000000 --- a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/cmkem_comm_http.cpp +++ /dev/null @@ -1,363 +0,0 @@ - -/* - * Copyright (c) 2020 Huawei Technologies Co.,Ltd. - * - * openGauss is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * - * http://license.coscl.org.cn/MulanPSL2 - * - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - * ------------------------------------------------------------------------- - * - * cmkem_comm_http.cpp - * - * IDENTIFICATION - * src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/cmkem_comm_http.cpp - * - * ------------------------------------------------------------------------- - */ - -#include "pg_config.h" -#include "cmkem_comm_http.h" - -#include -#include -#include -#include "curl/curl.h" -#include "cjson/cJSON.h" - -#ifdef ENABLE_UT -#define static -#endif - -static size_t dump_http_response_msg_callback(void *tmp_cur_res_str, size_t chr_size, size_t cur_res_size, - void *tmp_http_res_list) -{ - bool is_need_dump = false; - HttpResList *http_res_list = NULL; - - if (tmp_cur_res_str == NULL || tmp_http_res_list == NULL) { - return 0; - } - - http_res_list = (HttpResList *)tmp_http_res_list; - if (http_res_list->is_stop_dump) { - return chr_size * cur_res_size; - } - - const char *cur_res_str = (const char *)tmp_cur_res_str; - if (http_res_list->filter_res_type == HTTP_MSG) { - is_need_dump = true; - } else { - switch (http_res_list->cur_pos) { - case HTTP_RESLINE: - if (http_res_list->filter_res_type == HTTP_RESLINE) { - is_need_dump = true; - http_res_list->is_stop_dump = true; - } - if (http_res_list->loop_cnt == 2) { - http_res_list->cur_pos = HTTP_RESHEADER; - } - http_res_list->loop_cnt++; - break; - case HTTP_RESHEADER: - if (http_res_list->filter_res_type == HTTP_RESHEADER) { - is_need_dump = true; - } - if (strlen(cur_res_str) == strlen("\r\n") && cur_res_str[0] == '\r' && cur_res_str[1] == '\n') { - http_res_list->cur_pos = HTTP_RESBODY; - } - break; - case HTTP_RESBODY: - if (http_res_list->filter_res_type == HTTP_RESBODY) { - is_need_dump = true; - } - break; - default: - break; - } - } - - if (is_need_dump) { - cmkem_list_append(http_res_list->str_list, cur_res_str); - } - - return chr_size * cur_res_size; -} - -/* any traversal method is ok, we use inorder travers here */ -CmkemErrCode traverse_jsontree_with_raplace_value(cJSON *json_tree, ReplaceJsonTempValue replace_rules[], - size_t rule_cnt) -{ - char *new_value = NULL; - CmkemErrCode ret = CMKEM_SUCCEED; - - if (json_tree == NULL) { - return CMKEM_SUCCEED; - } - - if (cJSON_IsString(json_tree)) { - for (size_t i = 0; i < rule_cnt; i++) { - if (strcmp(replace_rules[i].src_value, cJSON_GetStringValue(json_tree)) == 0) { - new_value = cJSON_SetValuestring(json_tree, replace_rules[i].dest_value); - if (new_value == NULL) { - cmkem_errmsg("failed to set the value of json tree."); - return CMKEM_SET_CJSON_VALUE_ERR; - } - } - } - } - - ret = traverse_jsontree_with_raplace_value(json_tree->next, replace_rules, rule_cnt); - check_cmkem_ret(ret); - - return traverse_jsontree_with_raplace_value(json_tree->child, replace_rules, rule_cnt); -} - -CmkemErrCode traverse_jsontree_with_get_value(cJSON *json_tree, const char **key_list, CmkemStrList *value_list) -{ - CmkemErrCode ret = CMKEM_SUCCEED; - - /* we can make sure key_list[TAIL] == NULL */ - if (json_tree == NULL || key_list[0] == NULL) { - return CMKEM_SUCCEED; - } - - if (json_tree->string != NULL && strcmp(key_list[0], json_tree->string) == 0) { - cmkem_list_append(value_list, cJSON_GetStringValue(json_tree)); - return traverse_jsontree_with_get_value(json_tree, key_list + 1, value_list); - } - - ret = traverse_jsontree_with_get_value(json_tree->next, key_list, value_list); - check_cmkem_ret(ret); - - return traverse_jsontree_with_get_value(json_tree->child, key_list, value_list); -} - -CmkemErrCode set_http_config(CURL *http_obj, HttpConfig *http_config) -{ - CURLcode curl_ret = CURLE_OK; - - curl_ret = curl_easy_setopt(http_obj, CURLOPT_TIMEOUT, http_config->timeout); - check_curl_ret(curl_ret); - - curl_ret = curl_easy_setopt(http_obj, CURLOPT_SSL_VERIFYPEER, false); - check_curl_ret(curl_ret); - - curl_ret = curl_easy_setopt(http_obj, CURLOPT_SSL_VERIFYHOST, false); - check_curl_ret(curl_ret); - - return CMKEM_SUCCEED; -} - -CmkemErrCode set_http_reqline(CURL *http_obj, HttpMethod method, const char *url, char *version) -{ - CURLcode curl_ret = CURLE_OK; - - if (method == HTTP_POST) { - curl_ret = curl_easy_setopt(http_obj, CURLOPT_HTTPPOST, 1); - } else { - cmkem_errmsg("invalid http method, only support HTTP REQUEST now."); - return CMKEM_CURL_INIT_ERR; - } - - curl_ret = curl_easy_setopt(http_obj, CURLOPT_URL, url); - check_curl_ret(curl_ret); - - return CMKEM_SUCCEED; -} - -CmkemErrCode set_http_reqheader(CURL *http_obj, const char *http_heaser_list[], - struct curl_slist **ret_header_list) -{ - CURLcode curl_ret = CURLE_OK; - - for (size_t i = 0; http_heaser_list[i] != NULL; i++) { - *ret_header_list = curl_slist_append(*ret_header_list, http_heaser_list[i]); - if (*ret_header_list == NULL) { - cmkem_errmsg("failed to set http header: '%s'.", http_heaser_list[i]); - return CMKEM_CURL_ERR; - } - } - - curl_ret = curl_easy_setopt(http_obj, CURLOPT_HTTPHEADER, *ret_header_list); - check_curl_ret(curl_ret); - - return CMKEM_SUCCEED; -} - -CmkemErrCode set_http_reqbody(CURL *http_obj, const char *http_body) -{ - CURLcode curl_ret = curl_easy_setopt(http_obj, CURLOPT_POSTFIELDS, http_body); - check_curl_ret(curl_ret); - - return CMKEM_SUCCEED; -} - -CmkemErrCode get_http_resmsg(CURL *http_obj, HttpResListType res_part, CmkemStrList **response_msg, long *stat_code) -{ - HttpResList http_res_list = {"", res_part, HTTP_RESLINE, NULL, 0, false, 0}; - long http_stat_code = 0; - CURLcode curl_ret = CURLE_OK; - - http_res_list.str_list = malloc_cmkem_list(); - if (http_res_list.str_list == NULL) { - return CMKEM_MALLOC_MEM_ERR; - } - - curl_easy_setopt(http_obj, CURLOPT_HEADER, 1); /* set the localtion of write_data */ - - curl_easy_setopt(http_obj, CURLOPT_WRITEFUNCTION, dump_http_response_msg_callback); - - curl_easy_setopt(http_obj, CURLOPT_WRITEDATA, (void *) &http_res_list); - - curl_easy_setopt(http_obj, CURLOPT_FOLLOWLOCATION, 1); - - curl_ret = curl_easy_perform(http_obj); - if (curl_ret != CURLE_OK) { - cmkem_list_free(http_res_list.str_list); - cmkem_errmsg("curl error. err code: '%lu', err msg: '%s'.", curl_ret, curl_easy_strerror(curl_ret)); - return CMKEM_CURL_ERR; - } - - curl_ret = curl_easy_getinfo(http_obj, CURLINFO_RESPONSE_CODE, &http_stat_code); - if (curl_ret != CURLE_OK) { - cmkem_list_free(http_res_list.str_list); - cmkem_errmsg("curl error. err code: '%lu', err msg: '%s'.", curl_ret, curl_easy_strerror(curl_ret)); - return CMKEM_CURL_ERR; - } - - *response_msg = http_res_list.str_list; - *stat_code = http_stat_code; - return CMKEM_SUCCEED; -} - -CmkemErrCode http_request(HttpReqMsg* http_req_msg, HttpConfig *http_config, CmkemStrList **http_res_list, - HttpStatusCode *stat_code) -{ - CURL *sender = NULL; - struct curl_slist *http_header = NULL; - long http_stat_code = 0; - CmkemErrCode ret = CMKEM_SUCCEED; - - curl_global_init(CURL_GLOBAL_ALL); - sender = curl_easy_init(); - if (sender == NULL) { - return CMKEM_CURL_ERR; - } - - ret = set_http_config(sender, http_config); - if (ret != CMKEM_SUCCEED) { - curl_easy_cleanup(sender); - return ret; - } - - ret = set_http_reqline(sender, http_req_msg->method, http_req_msg->url, http_req_msg->version); - if (ret != CMKEM_SUCCEED) { - curl_easy_cleanup(sender); - return ret; - } - - ret = set_http_reqheader(sender, http_req_msg->header_list, &http_header); - if (ret != CMKEM_SUCCEED) { - curl_slist_free_all(http_header); - curl_easy_cleanup(sender); - return ret; - } - - ret = set_http_reqbody(sender, http_req_msg->body); - if (ret != CMKEM_SUCCEED) { - curl_slist_free_all(http_header); - curl_easy_cleanup(sender); - return ret; - } - - ret = get_http_resmsg(sender, http_config->res_part, http_res_list, &http_stat_code); - curl_slist_free_all(http_header); - curl_easy_cleanup(sender); - if (ret != CMKEM_SUCCEED) { - return ret; - } - - curl_global_cleanup(); - *stat_code = (HttpStatusCode)http_stat_code; - - return CMKEM_SUCCEED; -} - -char *make_content_length(const char *reqbody) -{ - errno_t rc = 0; - - char *reqhdr = (char *)malloc(MAX_HDR_LEN); - if (reqhdr == NULL) { - return NULL; - } - - rc = sprintf_s(reqhdr, MAX_HDR_LEN, "Content-Length: %lu", strlen(reqbody)); - securec_check_ss_c(rc, "", ""); - - return reqhdr; -} - -char *find_resheader(CmkemStrList *resheader_list, const char *resheader_type) -{ - char *ret = NULL; - - for (size_t i = 0; i < cmkem_list_len(resheader_list); i++) { - CmkemStrList *cur_header = cmkem_split(cmkem_list_val(resheader_list, i), ':'); - if (cur_header == NULL) { - return NULL; - } - - /* the cur_header should be like ["$key", "$value"] */ - if (cmkem_list_len(cur_header) > 1 && strcmp(resheader_type, cmkem_list_val(cur_header, 0)) == 0) { - ret = cmkem_list_val(cur_header, 1); - free_cmkem_list_with_skip(cur_header, 1); - break; - } else { - cmkem_list_free(cur_header); - } - } - - return ret; -} - -CmkemErrCode find_value_from_http_resbody(const char *http_resbody, const char **key_list, - CmkemStrList **value_list) -{ - size_t key_cnt = 0; - - cJSON *json_tree = cJSON_Parse(http_resbody); - if (json_tree == NULL) { - cmkem_errmsg("fail to convert http response body to cjson tree."); - return CMKEM_CJSON_PARSE_ERR; - } - - *value_list = malloc_cmkem_list(); - if (*value_list == NULL) { - cJSON_Delete(json_tree); - return CMKEM_MALLOC_MEM_ERR; - } - - for (size_t i = 0; key_list[i] != NULL; i++) { - key_cnt++; - } - - /* we will check 'key_cn' and 'value_cnt', so, the return value of this func is unimportant */ - (void)traverse_jsontree_with_get_value(json_tree, key_list, *value_list); - cJSON_Delete(json_tree); - - if (key_cnt != cmkem_list_len(*value_list)) { - cmkem_errmsg("%u keys are given, but only find %u values from list.", key_cnt, cmkem_list_len(*value_list)); - cmkem_list_free(*value_list); - return CMKEM_FIND_CSJON_ERR; - } - - return CMKEM_SUCCEED; -} diff --git a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/cmkem_comm_http.h b/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/cmkem_comm_http.h deleted file mode 100644 index f8d862af1..000000000 --- a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/cmkem_comm_http.h +++ /dev/null @@ -1,149 +0,0 @@ - -/* - * Copyright (c) 2020 Huawei Technologies Co.,Ltd. - * - * openGauss is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * - * http://license.coscl.org.cn/MulanPSL2 - * - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - * ------------------------------------------------------------------------- - * - * cmkem_comm_http.h - * - * IDENTIFICATION - * src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/cmkem_comm_http.h - * - * ------------------------------------------------------------------------- - */ - -#ifndef CMKEM_COMM_HTTP_H -#define CMKEM_COMM_HTTP_H - -#include "cmkem_comm.h" -#include "cjson/cJSON.h" -#include "curl/curl.h" - -const int MAX_URL_BUF_LEN = 256; -const int MAX_HDR_LEN = 64; - -typedef enum { - HTTP_POST, - HTTP_GET, -} HttpMethod; - -typedef enum { - HTTP_MSG = 0, - HTTP_RESLINE, - HTTP_RESHEADER, - HTTP_RESBODY, -} HttpResListType; - -typedef enum { - HTTP_OK = 200, - HTTP_ACCEPT = 202, - HTTP_NO_CONTENT = 204, - HTTP_MULTIPLE_CHICES = 300, - HTTP_BAD_REQUEST = 400, - HTTP_UNAUTHORIZED = 401, - HTTP_FORBIDDEN = 403, - HTTP_NOT_FOUND = 404, - HTTP_METHOD_NOT_ALLOWED = 405, - HTTP_NOT_ACCEPTABLE = 406, - HTTP_PROXY_AUTHENTICATION_REQUIRED = 407, - HTTP_REQUEST_TIMEOUT = 408, - HTTP_CONFLICT = 409, - HTTP_INTERNAL_SERVER_ERROR = 500, - HTTP_NOT_IMPLEMENTED = 501, - HTTP_BAD_GATEWAY = 502, - HTTP_SERVICE_UNAVAILABLE = 503, - HTTP_GATEWAY_TIMEOUT = 504, -} HttpStatusCode; - -/* - * it's hard to accept the param order of this struct HttpReqMsg is different from that of HTTP message - * however, only put the string array(param header_list) in the tail of the struct, can it be initialized - */ -typedef struct HttpReqMsg { - /* request line */ - HttpMethod method; - const char *url; - char *version; - - /* request body */ - const char *body; - - /* request header */ - const char **header_list; -} HttpReqMsg; - -typedef struct HttpResMsg { - /* reponse line */ - char *version; - HttpStatusCode status_code; - - /* reponse header */ - CmkemStrList *header_list; - - /* reponse body */ - char *body; -} HttpResMsg; - -typedef struct HttpConfig { - int timeout; - HttpResListType res_part; -} HttpConfig; - -/* - * the http response message include (1 response line (2 response header (3 response body - * @ param filter_res_type - only dump messages of the specified type - * @ param cur_pos - record current position of http resonse message - * @ param str_list - dump all message to string list - */ -typedef struct HttpResStrList { - const char *remote_server; - HttpResListType filter_res_type; - HttpResListType cur_pos; - CmkemStrList *str_list; - size_t node_cnt; - bool is_stop_dump; - - size_t loop_cnt; -} HttpResList; - -typedef struct { - const char *src_value; - const char *dest_value; -} ReplaceJsonTempValue; - -#define check_curl_ret(curl_ret) \ - do { \ - if ((curl_ret) != CURLE_OK) { \ - cmkem_errmsg("curl error. err code: '%lu', err msg: '%s.", (curl_ret), curl_easy_strerror((curl_ret))); \ - return CMKEM_CURL_ERR; \ - } \ - } while (0) - -extern CmkemErrCode traverse_jsontree_with_raplace_value(cJSON *json_tree, ReplaceJsonTempValue replace_rules[], - size_t rule_cnt); -extern CmkemErrCode traverse_jsontree_with_get_value(cJSON *json_tree, const char **key_list, CmkemStrList *value_list); -extern CmkemErrCode set_http_config(CURL *http_obj, HttpConfig *http_config); -extern CmkemErrCode set_http_reqline(CURL *http_obj, HttpMethod method, const char *url, char *version); -extern CmkemErrCode set_http_reqheader(CURL *http_obj, const char *http_heaser_list[], - struct curl_slist* ret_header_list); -extern CmkemErrCode set_http_reqbody(CURL *http_obj, const char *http_body); -extern CmkemErrCode get_http_resmsg(CURL *http_obj, HttpResListType res_part, CmkemStrList **response_msg, - long *stat_code); -extern CmkemErrCode http_request(HttpReqMsg* http_req_msg, HttpConfig *http_config, CmkemStrList **http_res_list, - HttpStatusCode *stat_code); -extern char *find_resheader(CmkemStrList *resheader_list, const char *resheader_type); -extern char *make_content_length(const char *reqbody); -extern CmkemErrCode find_value_from_http_resbody(const char *http_resbody, const char **key_list, - CmkemStrList **value_list); - -#endif /* CMKEM_COMM_HTTP_H */ diff --git a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/kms_restful_temp.ini b/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/kms_restful_temp.ini deleted file mode 100644 index e8b3136ad..000000000 --- a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/kms_restful_temp.ini +++ /dev/null @@ -1,102 +0,0 @@ -#define JS_TO_STR(R) #R - -const char *temp_iam_auth_req = JS_TO_STR(( - { - "auth": { - "identity": { - "methods": ["password"], - "password": { - "user": { - "name": "$user_name$", - "password": "$password$", - "domain": { - "name": "$domain_name$" - } - } - } - }, - "scope": { - "project": { - "name": "$project_name$" - } - } - } - } -)); - -const char *temp_kms_select_key_req = JS_TO_STR(( - { - "key_id": "$cmk_id$" - } -)); - -const char *temp_kms_select_key_res = JS_TO_STR(( - { - "key_info": { - "key_id": "0d0466b0-e727-4d9c-b35d-f84bb474a37f", - "domain_id": "b168fe00ff56492495a7d22974df2d0b", - "key_alias": "kms_test", - "realm": "aaa", - "key_description": "", - "creation_date": "1472442386000", - "scheduled_deletion_date": "", - "key_state": "2", - "default_key_flag": "0", - "key_type": "1", - "expiration_time":"1501578672000", - "origin":"kms", - "key_rotation_enabled":"false", - "sys_enterprise_project_id ": "0" - } - } -)); - -const char *temp_kms_enc_key_req = JS_TO_STR(( - { - "key_id": "$cmk_id$", - "plain_text":"$cek_plain$", - "datakey_plain_length": "$cek_plain_len$" - } -)); - -const char *temp_kms_enc_key_res = JS_TO_STR(( - { - "key_id": "0d0466b0-e727-4d9c-b35d-f84bb474a37f", - "plain_text":"00000000000000000079B43003D2320D9F0E8EA9831A92759FB4B", - "datakey_plain_length": "64" - } -)); - -const char *temp_kms_dec_key_req = JS_TO_STR(( - { - "key_id": "$cmk_id$", - "cipher_text":"$cek_cipher$", - "datakey_cipher_length": "$cek_cipher_len$" - } -)); - -const char *temp_kms_dec_key_res = JS_TO_STR(( - { - "key_id": "0d0466b0-e727-4d9c-b35d-f84bb474a37f", - "cipher_text":"00000000000098EF6ED309979B43003D2320D9F0E8EA9831A92759FB4B", - "datakey_cipher_length": "64" - } -)); - -const char *temp_kms_err_res = JS_TO_STR(( - { - "error": { - "error_code": "KMS.XXXX", - "error_msg": "XXX" - } - } -)); - -const char *temp_iam_err_res = JS_TO_STR(( - { - "error": { - "message": "The request you have made requires authentication.", - "title": "Unauthorized" - } - } -)); \ No newline at end of file diff --git a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/reg_cmkem_manager_main.cpp b/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/reg_cmkem_manager_main.cpp deleted file mode 100644 index 33b8f9c4d..000000000 --- a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/reg_cmkem_manager_main.cpp +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright (c) 2020 Huawei Technologies Co.,Ltd. - * - * openGauss is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * - * http://license.coscl.org.cn/MulanPSL2 - * - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - * ------------------------------------------------------------------------- - * - * reg_cmkem_manager_main.cpp - * you can register your own key management tool/component/service here. - * what's more, you can choose which to use according to the version and usage scenarios. - * - * IDENTIFICATION - * src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/reg_cmkem_manager_main.cpp - * - * ------------------------------------------------------------------------- - */ - - -#include "reg_cmkem_manager_main.h" - -#include - -#include "cmkem_version_control.h" -#include "register_gs_ktool.h" -#include "register_huawei_kms.h" -#include "register_local_kms.h" -#include "register_gram_check.h" - -static bool has_reg_hookfunc = false; -static pthread_mutex_t reg_lock = PTHREAD_MUTEX_INITIALIZER; - -CmkemErrCode reg_all_cmk_entity_manager() -{ - if (pthread_mutex_lock(®_lock) != 0) { - return CMKEM_REG_CMK_MANAGER_ERR; - } - - if (has_reg_hookfunc) { - pthread_mutex_unlock(®_lock); - return CMKEM_SUCCEED; - } - -#ifdef ENABLE_GS_KTOOL - if (reg_cmke_manager_gs_ktool_main() < 0) { - goto err_with_unlock; - } -#endif - -#ifdef ENABLE_HUAWEI_KMS - if (reg_cmke_manager_huwei_kms_main() < 0) { - goto err_with_unlock; - } -#endif - -#ifdef ENABLE_LOCAL_KMS - if (reg_cmke_manager_local_kms_main() < 0) { - goto err_with_unlock; - } -#endif - - /* - * register new key management tool/component/service here - */ - - if (reg_grammar_check_main() < 0) { - goto err_with_unlock; - } - - has_reg_hookfunc = true; - pthread_mutex_unlock(®_lock); - - return CMKEM_SUCCEED; - -err_with_unlock: - pthread_mutex_unlock(®_lock); - return CMKEM_REG_CMK_MANAGER_ERR; -} - -void exit_all_cmk_entity_manager() -{ - if (pthread_mutex_lock(®_lock) != 0) { - return; - } - - if (!has_reg_hookfunc) { - pthread_mutex_unlock(®_lock); - return; - } - -#ifdef ENABLE_GS_KTOOL - exit_cmke_manager_gs_ktool(); -#endif - - pthread_mutex_unlock(®_lock); -} diff --git a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/reg_hook_frame.cpp b/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/reg_hook_frame.cpp deleted file mode 100644 index bb7e15353..000000000 --- a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/reg_hook_frame.cpp +++ /dev/null @@ -1,190 +0,0 @@ -/* - * Copyright (c) 2020 Huawei Technologies Co.,Ltd. - * - * openGauss is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * - * http://license.coscl.org.cn/MulanPSL2 - * - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - * ------------------------------------------------------------------------- - * - * reg_hook_frame.cpp - * we use 3 level key encryption scheme in Client Encrypt: - * 1. use CEK (Column Encryption Key) to encrypt data - * 2. use CMK (Client Master Key) to encrypt CEK - * 3. use RK (Root Key) to encrypt CMK - * for CMK, we use CMKO and CMKE to describe it: - * 1. CMKO: CMK object, create by {SQL: CREATE CLIENT MASTER KEY ...} - * 2. CMKE: the actual CMK entity, managed by thierd party key management tool/component/service - * the CMKO only stores the method of reading and use CMKE. - * so, while creating CMKO, you need to use KEY_STORE and KEY_PATH to specify how to read and use CMKE. - * - * this part provides a flexible hook machanish to support users to register and manage their own CMKE by their - * key management tool/component/service. - * - * if you are familiar with Linux Netfilter or SELiunx, you will know the code logic of this part well soon. - * - * IDENTIFICATION - * src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/reg_hook_frame.cpp - * - * ------------------------------------------------------------------------- - */ - -#include "reg_hook_frame.h" - -#include -#include "cmkem_comm.h" - -static CmkEntityManager cmk_entity_manager_list[MAX_CMK_MANAGER_NUM] = {0}; -size_t cmke_manager_cnt = 0; - -CmkemErrCode reg_cmk_entity_manager(CmkEntityManager cmke_manager) -{ - if (cmke_manager_cnt >= MAX_CMK_MANAGER_NUM) { - cmkem_errmsg("the number of cmk managers has reached the maximum: %d.", MAX_CMK_MANAGER_NUM); - return CMKEM_REGISTRE_FUNC_ERR; - } - - cmk_entity_manager_list[cmke_manager_cnt] = cmke_manager; - cmke_manager_cnt++; - return CMKEM_SUCCEED; -} - -/* privided to client */ -CmkemErrCode create_cmk_obj(CmkIdentity *cmk_identity) -{ - ProcessPolicy policy = POLICY_CONTINUE; - - if (cmke_manager_cnt == 0) { - cmkem_errmsg("in hook point: CREATE_CMK_OBJ, no registered hook function found."); - return CMKEM_REGISTRE_FUNC_ERR; - } - - for (size_t i = 0; i < cmke_manager_cnt; i++) { - if (cmk_entity_manager_list[i].crt_cmko_hookfunc == NULL) { - continue; - } - - policy = cmk_entity_manager_list[i].crt_cmko_hookfunc(cmk_identity); - if (policy == POLICY_BREAK) { - return CMKEM_SUCCEED; - } else if (policy == POLICY_ERROR) { - return CMKEM_CREATE_CMK_ERR; - } - /* else continue */ - } - - return CMKEM_SUCCEED; -} - -/* privided to client */ -CmkemErrCode encrypt_cek_plain(CmkemUStr *cek_plain, CmkIdentity *cmk_identity, CmkemUStr **cek_cipher) -{ - ProcessPolicy policy = POLICY_CONTINUE; - - if (cmke_manager_cnt == 0) { - cmkem_errmsg("in hook point: ENCRYPT_CEK_ENTITY, no registered hook function found."); - return CMKEM_REGISTRE_FUNC_ERR; - } - - for (size_t i = 0; i < cmke_manager_cnt; i++) { - if (cmk_entity_manager_list[i].enc_cek_cipher_hookfunc == NULL) { - continue; - } - - policy = cmk_entity_manager_list[i].enc_cek_cipher_hookfunc(cek_plain, cmk_identity, cek_cipher); - if (policy == POLICY_BREAK) { - return CMKEM_SUCCEED; - } else if (policy == POLICY_ERROR) { - return CMKEM_ENCRYPT_CEK_ERR; - } - /* else continue */ - } - - return CMKEM_SUCCEED; -} - -/* privided to client */ -CmkemErrCode decrypt_cek_cipher(CmkemUStr *cek_cipher, CmkIdentity *cmk_identity, CmkemUStr **cek_plain) -{ - ProcessPolicy policy = POLICY_CONTINUE; - - if (cmke_manager_cnt == 0) { - cmkem_errmsg("in hook point: DECRYPT_CEK_ENTITY, no registered hook function found."); - return CMKEM_REGISTRE_FUNC_ERR; - } - - for (size_t i = 0; i < cmke_manager_cnt; i++) { - if (cmk_entity_manager_list[i].dec_cek_plain_hookfunc == NULL) { - continue; - } - - policy = cmk_entity_manager_list[i].dec_cek_plain_hookfunc(cek_cipher, cmk_identity, cek_plain); - if (policy == POLICY_BREAK) { - return CMKEM_SUCCEED; - } else if (policy == POLICY_ERROR) { - return CMKEM_DECRYPT_CEK_ERR; - } - /* else continue */ - } - - return CMKEM_SUCCEED; -} - -/* privided to client */ -CmkemErrCode drop_cmk_obj(CmkIdentity *cmk_identity) -{ - ProcessPolicy policy = POLICY_CONTINUE; - - if (cmke_manager_cnt == 0) { - cmkem_errmsg("in hook point: DROP_CMK_OBJ, no registered hook function found."); - return CMKEM_REGISTRE_FUNC_ERR; - } - - for (size_t i = 0; i < cmke_manager_cnt; i++) { - if (cmk_entity_manager_list[i].drop_cmko_hookfunc == NULL) { - continue; - } - - policy = cmk_entity_manager_list[i].drop_cmko_hookfunc(cmk_identity); - if (policy == POLICY_BREAK) { - return CMKEM_SUCCEED; - } else if (policy == POLICY_ERROR) { - return CMKEM_DROP_CMK_ERR; - } - /* else continue */ - } - - return CMKEM_SUCCEED; -} - -CmkemErrCode post_create_cmk_obj(CmkIdentity *cmk_identity) -{ - ProcessPolicy policy = POLICY_CONTINUE; - - if (cmke_manager_cnt == 0) { - cmkem_errmsg("in hook point: POST_CREATE_CMK_OBJ, no registered hook function found."); - return CMKEM_REGISTRE_FUNC_ERR; - } - - for (size_t i = 0; i < cmke_manager_cnt; i++) { - if (cmk_entity_manager_list[i].post_crt_cmko_hookfunc == NULL) { - continue; - } - - policy = cmk_entity_manager_list[i].post_crt_cmko_hookfunc(cmk_identity); - if (policy == POLICY_BREAK) { - return CMKEM_SUCCEED; - } else if (policy == POLICY_ERROR) { - return CMKEM_CREATE_CMK_ERR; - } - /* else continue */ - } - - return CMKEM_SUCCEED; -} diff --git a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/reg_hook_frame.h b/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/reg_hook_frame.h deleted file mode 100644 index a80b060e9..000000000 --- a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/reg_hook_frame.h +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (c) 2020 Huawei Technologies Co.,Ltd. - * - * openGauss is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * - * http://license.coscl.org.cn/MulanPSL2 - * - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - * ------------------------------------------------------------------------- - * - * reg_hook_frame.h - * we use 3 level key encryption scheme in Client Encrypt: - * 1. use CEK (Column Encryption Key) to encrypt data - * 2. use CMK (Client Master Key) to encrypt CEK - * 3. use RK (Root Key) to encrypt CMK - * for CMK, we use CMKO and CMKE to describe it: - * 1. CMKO: CMK object, create by {SQL: CREATE CLIENT MASTER KEY ...} - * 2. CMKE: the actual CMK entity, managed by thierd party key management tool/component/service - * the CMKO only stores the method of reading and use CMKE. - * so, while creating CMKO, you need to use KEY_STORE and KEY_PATH to specify how to read and use CMKE. - * - * this part provides a flexible hook machanish to support users to register and manage their own CMKE by their - * key management tool/component/service. - * - * if you are familiar with Linux Netfilter or SELiunx, you will know the code logic of this part well soon. - * - * IDENTIFICATION - * src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/reg_hook_frame.h - * - * ------------------------------------------------------------------------- - */ - -#ifndef REG_HOOK_FRAME_H -#define REG_HOOK_FRAME_H - -#include "cmkem_comm.h" - -const int MAX_CMK_MANAGER_NUM = 20; -/* - * if more than one key management tool/component/service is registered, all of them can handle the same SQL syntax - * so, user need to coordinate how they handle it through the return value: ProcessPolicy - */ -typedef enum { - POLICY_CONTINUE, - POLICY_BREAK, - POLICY_ERROR, - PLLICY_PROCESSED, -} ProcessPolicy; - -/* called while process {SQL: CREATE CLIENT MASTER KEY ...} */ -/* called while process {SQL:CREATE COLUMN ENTRYPTION KEY ...} */ -/* called while process {SQL: INSERT/UPDATE/SELECT encrypted_column ...} */ -/* called while process {SQL: DROP CLIENT MASTER KEY ...} */ -typedef ProcessPolicy (*CreateCmkObjectHookFunc)(CmkIdentity *cmk_identity); -typedef ProcessPolicy (*EncryptCekPlainHookFunc)(CmkemUStr *cek_plain, CmkIdentity *cmk_identity, - CmkemUStr **cek_cipher); -typedef ProcessPolicy (*DecryptCekCipherHookFunc)(CmkemUStr *cek_cipher, CmkIdentity *cmk_identity, - CmkemUStr **cek_plain); -typedef ProcessPolicy (*DropCmKObjectHookFunc)(CmkIdentity *cmk_identity); -typedef ProcessPolicy (*PostCreateCmkObjectHookFunc)(CmkIdentity *cmk_identity); - -typedef struct CmkEntityManager { - CreateCmkObjectHookFunc crt_cmko_hookfunc; - EncryptCekPlainHookFunc enc_cek_cipher_hookfunc; - DecryptCekCipherHookFunc dec_cek_plain_hookfunc; - DropCmKObjectHookFunc drop_cmko_hookfunc; - PostCreateCmkObjectHookFunc post_crt_cmko_hookfunc; -} CmkEntityManager; - -/* privided to users for registering their own key management tool/component/service */ -CmkemErrCode reg_cmk_entity_manager(CmkEntityManager cmke_manager); - -/* privided to client for calling users callback function */ -CmkemErrCode create_cmk_obj(CmkIdentity *cmk_identity); -CmkemErrCode encrypt_cek_plain(CmkemUStr *cek_plain, CmkIdentity *cmk_identity, CmkemUStr **cek_cipher); -CmkemErrCode decrypt_cek_cipher(CmkemUStr *cek_cipher, CmkIdentity *cmk_identity, CmkemUStr **cek_plain); -CmkemErrCode drop_cmk_obj(CmkIdentity *cmk_identity); -CmkemErrCode post_create_cmk_obj(CmkIdentity *cmk_identity); - -#endif /* REG_HOOK_FRAME_H */ diff --git a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/register_gram_check.cpp b/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/register_gram_check.cpp deleted file mode 100644 index f629e3ddb..000000000 --- a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/register_gram_check.cpp +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (c) 2020 Huawei Technologies Co.,Ltd. - * - * openGauss is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * - * http://license.coscl.org.cn/MulanPSL2 - * - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - * ------------------------------------------------------------------------- - * - * register_gram_check.cpp - * we support users to register any number of key management tool/components/services, all of which - * have the right to process the CREATE CMKO statement. - * if a CREATE CMKO statement is not processed at all, we agree that it's a wrong and unexpected statement. - * so, we should put this grammar check at the end of hook function list, to make sure the unexpected SQL statement - * can be reported. - * - * IDENTIFICATION - * src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/register_gram_check.cpp - * - * ------------------------------------------------------------------------- - */ - -#include "register_gram_check.h" -#include "reg_hook_frame.h" - -static ProcessPolicy create_cmk_obj_hookfunc(CmkIdentity *cmk_identity) -{ - if (cmk_identity->cmk_store == NULL) { - cmkem_errmsg("failed to create client master key, failed to find arg: KEY_STORE."); - } else { - cmkem_errmsg("key store type '%s' is not supported.", cmk_identity->cmk_store); - } - - return POLICY_ERROR; -} - -static ProcessPolicy encrypt_cek_plain_hookfunc(CmkemUStr *cek_plain, CmkIdentity *cmk_identity, CmkemUStr **cek_cipher) -{ - cmkem_errmsg("failed to find client master key to encrypt column encryption key."); - return POLICY_ERROR; -} - -static ProcessPolicy decrypt_cek_cipher_hookfunc(CmkemUStr *cek_cipher, CmkIdentity *cmk_identity, - CmkemUStr **cek_plain) -{ - cmkem_errmsg("failed to find client master key to decrypt column encryption key."); - return POLICY_ERROR; -} - -int reg_grammar_check_main() -{ - CmkEntityManager gramm_check = { - create_cmk_obj_hookfunc, - encrypt_cek_plain_hookfunc, - decrypt_cek_cipher_hookfunc, - NULL, /* drop_cmk_obj_hook_func: no need */ - NULL, /* post_create_cmk_obj_hook_func: no need */ - }; - - return (reg_cmk_entity_manager(gramm_check) == CMKEM_SUCCEED) ? 0 : -1; -} diff --git a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/register_gram_check.h b/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/register_gram_check.h deleted file mode 100644 index ed3effb41..000000000 --- a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/register_gram_check.h +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2020 Huawei Technologies Co.,Ltd. - * - * openGauss is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * - * http://license.coscl.org.cn/MulanPSL2 - * - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - * ------------------------------------------------------------------------- - * - * register_gram_check.h - * we support users to register any number of key management tool/components/services, all of which - * have the right to process the CREATE CMKO statement. - * if a CREATE CMKO statement is not processed at all, we agree that it's a wrong and unexpected statement. - * so, we should put this grammar check at the end of hook function list, to make sure the unexpected SQL statement - * can be reported. - * - * IDENTIFICATION - * src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/register_gram_check.h - * - * ------------------------------------------------------------------------- - */ - -#ifndef REGISTER_GRAM_CHECK_H -#define REGISTER_GRAM_CHECK_H - -extern int reg_grammar_check_main(); - -#endif /* REG_GRAM_CHECK_H */ diff --git a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/register_huawei_kms.cpp b/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/register_huawei_kms.cpp deleted file mode 100644 index f97554cf1..000000000 --- a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/register_huawei_kms.cpp +++ /dev/null @@ -1,994 +0,0 @@ -/* - * Copyright (c) 2020 Huawei Technologies Co.,Ltd. - * - * openGauss is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * - * http://license.coscl.org.cn/MulanPSL2 - * - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - * ------------------------------------------------------------------------- - * - * register_huawei_kms.cpp - * Huawei KMS is an online key management service provided by Huawei Cloud. - * We support Huawei KMS to provide cmk entities for us, and we can send CEK entities - * to it for encrypting and decrypting - * - * IDENTIFICATION - * src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/register_huawei_kms.cpp - * - * ------------------------------------------------------------------------- - */ - -#include "cmkem_version_control.h" -#ifdef ENABLE_HUAWEI_KMS - -#include "register_huawei_kms.h" -#include -#include -#include -#include -#include -#include -#include -#include - -#include "cjson/cJSON.h" -#include "cmkem_comm.h" -#include "cmkem_comm_http.h" -#include "cmkem_comm_algorithm.h" -#include "reg_hook_frame.h" - -#include "./kms_restful_temp.ini" - -#ifdef ENABLE_UT -#define static -#endif - -typedef struct CachedAuthInfo { - char user_name[USER_NAME_BUF_LEN]; - char password[PASSWD_BUF_LEN]; - char domain_name[DOMAIN_NAME_BUF_LEN]; - char project_name[PROJ_NAME_BUF_LEN]; - char project_id[PROJ_ID_BUF_LEN]; - bool has_get_token; - char req_header_token[MAX_TOKEN_BUF_LEN]; /* X-Auth-Token:{TOKEN} */ -} CachedAuthInfo; - -typedef struct KmsInfo { - const char *kms_key; - char *kms_value; - size_t value_buf_len; -} KmsInfo; - -CachedAuthInfo *kms_cache_tbl[KMS_CACHE_TBL_LEN] = {0}; -static const char *supported_algorithms[] = {"AES_256", NULL}; - -static CachedAuthInfo *malloc_kms_cache(size_t cache_id); -static CmkemErrCode are_all_kms_infos_set(CachedAuthInfo *cache); -static void set_kms_cache_auth_token(CachedAuthInfo *cache, const char *token); -static CmkemErrCode get_kms_cache_ptr(size_t cache_id, CachedAuthInfo **cache); -static CmkemErrCode get_kms_token_from_iam(CachedAuthInfo *cache, char **token); - -static CmkemErrCode check_cmk_algo_validity(const char *cmk_algo); -static CmkemErrCode check_cmk_id_validity(CmkIdentity *cmk_identity); -static CmkemErrCode check_cmk_entity_validity(CmkIdentity *cmk_identity); -static CmkemErrCode check_plain_len_constant(CmkemUStr *plain); -static ProcessPolicy create_cmk_obj_hookfunc(CmkIdentity *cmk_identity); -static ProcessPolicy encrypt_cek_plain_hookfunc(CmkemUStr *cek_plain, CmkIdentity *cmk_identity, - CmkemUStr **cek_cipher); -static ProcessPolicy decrypt_cek_cipher_hookfunc(CmkemUStr *cek_cipher, CmkIdentity *cmk_identity, - CmkemUStr **cek_plain); - -/* - * these templates are HTTP request bodys of restful apis of Huawei Cloud IAM and KMS - * we need to use a definete json template, and pass it parameters such as user_name, password, project_name... - */ -cJSON *get_json_temp(KmsHttpMsgType json_tree_type) -{ - const char *temp_in = NULL; - cJSON *json_out = NULL; - - typedef struct { - KmsHttpMsgType type; - const char *temp; - } JsonTemp; - - const JsonTemp json_tbl[] = { - {IAM_AUTH_REQ, temp_iam_auth_req}, - {KMS_SELECT_CMK_REQ, temp_kms_select_key_req}, - {KMS_ENC_CEK_REQ, temp_kms_enc_key_req}, - {KMS_DEC_CEK_REQ, temp_kms_dec_key_req}, - }; - - temp_in = json_tbl[json_tree_type].temp; - - /* - * the format of the temp_in is : "(str_with_bracket)", we should remove the backet - * temp_in + 1 : remove the left bracket - * strlen(temp_in) - 2 : remove the right bracket - */ - json_out = cJSON_ParseWithLength(temp_in + 1, strlen(temp_in) - 2); - - return json_out; -} - -char *get_iam_auth_req_jsontemp(const char *user_name, const char *password, const char *domain_name, - const char *project_name) -{ - cJSON* json_temp = NULL; - char *json_temp_str = NULL; - ReplaceJsonTempValue replace_rules[] = { - {"$user_name$", user_name}, - {"$password$", password}, - {"$domain_name$", domain_name}, - {"$project_name$", project_name}, - }; - size_t rule_cnt = sizeof(replace_rules) / sizeof(replace_rules[0]); - CmkemErrCode ret = CMKEM_SUCCEED; - - json_temp = get_json_temp(IAM_AUTH_REQ); - if (json_temp == NULL) { - cmkem_errmsg("failed to get json template to construct the http message of IAM auth request."); - return NULL; - } - - ret = traverse_jsontree_with_raplace_value(json_temp, replace_rules, rule_cnt); - if (ret != CMKEM_SUCCEED) { - cJSON_Delete(json_temp); - return NULL; - } - - json_temp_str = cJSON_Print(json_temp); - cJSON_Delete(json_temp); - return json_temp_str; -} - -char *get_select_cmk_jsontemp(const char *key_id) -{ - cJSON* json_temp = NULL; - char *json_temp_str = NULL; - ReplaceJsonTempValue replace_rules[] = { - {"$cmk_id$", key_id}, - }; - size_t rule_cnt = sizeof(replace_rules) / sizeof(replace_rules[0]); - CmkemErrCode ret = CMKEM_SUCCEED; - - json_temp = get_json_temp(KMS_SELECT_CMK_REQ); - if (json_temp == NULL) { - cmkem_errmsg("failed to get json template to construct the http message of KMS select key request."); - return NULL; - } - - ret = traverse_jsontree_with_raplace_value(json_temp, replace_rules, rule_cnt); - if (ret != CMKEM_SUCCEED) { - cJSON_Delete(json_temp); - return NULL; - } - - json_temp_str = cJSON_Print(json_temp); - cJSON_Delete(json_temp); - return json_temp_str; -} - -char *get_enc_cek_plain_jsontemp(const char *cmk_id, const char *cek_plain, size_t cek_plain_len) -{ - cJSON* json_temp = NULL; - char *json_temp_str = NULL; - char cek_plain_len_str[ITOA_BUF_LEN] = {0}; - CmkemErrCode ret = CMKEM_SUCCEED; - itoa(cek_plain_len, cek_plain_len_str, ITOA_BUF_LEN); - ReplaceJsonTempValue replace_rules[] = { - {"$cmk_id$", cmk_id}, - {"$cek_plain$", cek_plain}, - {"$cek_plain_len$", cek_plain_len_str}, - }; - size_t rule_cnt = sizeof(replace_rules) / sizeof(replace_rules[0]); - - json_temp = get_json_temp(KMS_ENC_CEK_REQ); - if (json_temp == NULL) { - cmkem_errmsg("failed to get json template to construct the http message of IAM auth request."); - return NULL; - } - - ret = traverse_jsontree_with_raplace_value(json_temp, replace_rules, rule_cnt); - if (ret != CMKEM_SUCCEED) { - cJSON_Delete(json_temp); - return NULL; - } - - json_temp_str = cJSON_Print(json_temp); - cJSON_Delete(json_temp); - return json_temp_str; -} - -char *get_dec_cek_cipher_jsontemp(const char *cmk_id, const char *cek_cipher, size_t cek_cipher_len) -{ - cJSON* json_temp = NULL; - char cek_cipher_len_str[ITOA_BUF_LEN] = {0}; - CmkemErrCode ret = CMKEM_SUCCEED; - itoa(cek_cipher_len, cek_cipher_len_str, ITOA_BUF_LEN); - ReplaceJsonTempValue replace_rules[] = { - {"$cmk_id$", cmk_id}, - {"$cek_cipher$", cek_cipher}, - {"$cek_cipher_len$", cek_cipher_len_str}, - }; - size_t rule_cnt = sizeof(replace_rules) / sizeof(replace_rules[0]); - char *json_temp_str = NULL; - - json_temp = get_json_temp(KMS_DEC_CEK_REQ); - if (json_temp == NULL) { - cmkem_errmsg("failed to get json template to construct the http message of KMS decrypt key request."); - return NULL; - } - - ret = traverse_jsontree_with_raplace_value(json_temp, replace_rules, rule_cnt); - if (ret != CMKEM_SUCCEED) { - cJSON_Delete(json_temp); - return NULL; - } - - json_temp_str = cJSON_Print(json_temp); - cJSON_Delete(json_temp); - - return json_temp_str; -} - -void get_iam_url(const char *project_name, char *url_buf, size_t buf_len) -{ - errno_t rc = sprintf_s(url_buf, buf_len, "https://iam.%s.myhuaweicloud.com/v3/auth/tokens", project_name); - securec_check_ss_c(rc, "", ""); -} - -CmkemErrCode get_kms_url(size_t cache_id, KmsHttpMsgType kms_msg_type, char *url_buf, size_t url_buf_len) -{ - CmkemErrCode ret = CMKEM_SUCCEED; - char url_file_path_buf[MAX_URL_BUF_LEN] = {0}; - errno_t rc = 0; - CachedAuthInfo *cache = NULL; - - ret = get_kms_cache_ptr(cache_id, &cache); - check_cmkem_ret(ret); - - ret = get_url_file_path(kms_msg_type, cache->project_id, url_file_path_buf, MAX_URL_BUF_LEN); - check_cmkem_ret(ret); - - rc = sprintf_s(url_buf, url_buf_len, "https://kms.%s.myhuaweicloud.com%s", cache->project_name, url_file_path_buf); - securec_check_ss_c(rc, "", ""); - - return CMKEM_SUCCEED; -} - -CmkemErrCode get_url_file_path(KmsHttpMsgType kms_msg_type, const char *project_id, char *url_file_path_buf, - size_t buf_len) -{ - errno_t rc = 0; - - switch (kms_msg_type) { - case KMS_SELECT_CMK_REQ: - rc = sprintf_s(url_file_path_buf, buf_len, "/v1.0/%s/kms/describe-key", project_id); - securec_check_ss_c(rc, "", ""); - return CMKEM_SUCCEED; - case KMS_ENC_CEK_REQ: - rc = sprintf_s(url_file_path_buf, buf_len, "/v1.0/%s/kms/encrypt-datakey", project_id); - securec_check_ss_c(rc, "", ""); - return CMKEM_SUCCEED; - case KMS_DEC_CEK_REQ: - rc = sprintf_s(url_file_path_buf, buf_len, "/v1.0/%s/kms/decrypt-datakey", project_id); - securec_check_ss_c(rc, "", ""); - return CMKEM_SUCCEED; - default: - break; - } - - return CMKEM_UNKNOWN_ERR; -} - -CmkemErrCode get_kms_err_type(const char *kms_errmsg_body) -{ - const char *key_list[] = {"error_code", NULL}; - CmkemStrList *value_list = NULL; - CmkemErrCode ret = CMKEM_SUCCEED; - - if (kms_errmsg_body == NULL) { - return CMKEM_KMS_SERVER_ERR; - } - - ret = find_value_from_http_resbody(kms_errmsg_body, key_list, &value_list); - check_cmkem_ret(ret); - - if (strcasecmp(cmkem_list_val(value_list, 0), "kms.0303") == 0) { - cmkem_list_free(value_list); - return CMKEM_TOKEN_EXPIRED_ERR; - } - cmkem_list_free(value_list); - - return CMKEM_KMS_SERVER_ERR; -} - -static CachedAuthInfo *malloc_kms_cache(size_t cache_id) -{ - errno_t rc = 0; - - if (cache_id >= KMS_CACHE_TBL_LEN) { - return NULL; - } - - CachedAuthInfo *cache = (CachedAuthInfo *)malloc(sizeof(CachedAuthInfo)); - if (cache == NULL) { - return NULL; - } - - rc = memset_s(cache, sizeof(CachedAuthInfo), 0, sizeof(CachedAuthInfo)); - securec_check_c(rc, "", ""); - - kms_cache_tbl[cache_id] = cache; - - return cache; -} - -void free_kms_cache(size_t cache_id) -{ - if (cache_id >= KMS_CACHE_TBL_LEN) { - return; - } - - if (kms_cache_tbl[cache_id] == NULL) { - return; - } - - errno_t rc = memset_s(kms_cache_tbl[cache_id], sizeof(CachedAuthInfo), 0, sizeof(CachedAuthInfo)); - securec_check_c(rc, "", ""); - - cmkem_free(kms_cache_tbl[cache_id]); -} - -static CmkemErrCode are_all_kms_infos_set(CachedAuthInfo *cache) -{ - if (is_str_empty(cache->user_name) || - is_str_empty(cache->password) || - is_str_empty(cache->domain_name) || - is_str_empty(cache->project_name) || - is_str_empty(cache->project_id)) { - cmkem_errmsg("please make sure all Huawei IAM and KMS info have been set, including: " - "{iamUser, iamPassword, kmsDomain, kmsProjectName, kmsProjectId}."); - return CMKEM_CHECK_INPUT_AUTH_ERR; - } - - return CMKEM_SUCCEED; -} - -CmkemErrCode set_kms_cache_auth_info(size_t cache_id, const char *key, const char *value) -{ - CachedAuthInfo *cache = NULL; - CmkemErrCode ret = CMKEM_SUCCEED; - ret = get_kms_cache_ptr(cache_id, &cache); - if (ret == CMKEM_CACHE_IS_EMPTY) { - cache = malloc_kms_cache(cache_id); - if (cache == NULL) { - ret = CMKEM_MALLOC_MEM_ERR; - cmkem_errmsg("failed to malloc memory as client cache whose cache id is '%u'.", cache_id); - } else { - ret = CMKEM_SUCCEED; - } - } - check_cmkem_ret(ret); - - KmsInfo kms_info_map[] = { - {"iamUser", cache->user_name, USER_NAME_BUF_LEN}, - {"iamPassword", cache->password, PASSWD_BUF_LEN}, - {"kmsDomain", cache->domain_name, DOMAIN_NAME_BUF_LEN}, - {"kmsProjectName", cache->project_name, PROJ_NAME_BUF_LEN}, - {"kmsProjectId", cache->project_id, PROJ_ID_BUF_LEN}}; - errno_t rc = 0; - - if (key == NULL) { - cmkem_errmsg("the kms info should be in {iamUser, iamPassword, kmsDomain, kmsProjectName, kmsProjectId}."); - return CMKEM_CHECK_INPUT_AUTH_ERR; - } - - for (size_t i = 0; i < sizeof(kms_info_map) / sizeof(kms_info_map[0]); i++) { - if (strcmp(key, kms_info_map[i].kms_key) == 0) { - if (value == NULL || strlen(value) >= kms_info_map[i].value_buf_len) { - cmkem_errmsg("the value length of '%s' is invalid.", kms_info_map[i].kms_key); - return CMKEM_CHECK_INPUT_AUTH_ERR; - } - - rc = strcpy_s(kms_info_map[i].kms_value, kms_info_map[i].value_buf_len, value); - securec_check_c(rc, "", ""); - } - } - - return CMKEM_SUCCEED; -} - -static void set_kms_cache_auth_token(CachedAuthInfo *cache, const char *token) -{ - if (cache != NULL) { - errno_t rc = sprintf_s(cache->req_header_token, MAX_TOKEN_BUF_LEN, "X-Auth-Token:%s ", token); - securec_check_ss_c(rc, "", ""); - cache->has_get_token = true; - } -} - -static CmkemErrCode get_kms_cache_ptr(size_t cache_id, CachedAuthInfo **cache) -{ - if (cache_id >= KMS_CACHE_TBL_LEN) { - cmkem_errmsg("the client cahche id '%u' is invalid.", cache_id); - return CMKEM_CHECK_CHACHE_ID_ERR; - } - - if (kms_cache_tbl[cache_id] == NULL) { - cmkem_errmsg("the cache whose cache id is '%u' is empty.", cache_id); - return CMKEM_CACHE_IS_EMPTY; - } - - *cache = kms_cache_tbl[cache_id]; - return CMKEM_SUCCEED; -} - -CmkemErrCode get_kms_cache_token(size_t cache_id, char **req_token) -{ - CachedAuthInfo *cache = NULL; - CmkemErrCode ret = CMKEM_SUCCEED; - char *token = NULL; - - ret = get_kms_cache_ptr(cache_id, &cache); - if (ret == CMKEM_CACHE_IS_EMPTY) { - cmkem_errmsg("failed to get token when attempting to access the KMS server, please provide this information: " - "{iamUser, iamPassword, kmsDomain, kmsProjectName, kmsProjectId}."); - return CMKEM_GET_TOKEN_ERR; - } - check_cmkem_ret(ret); - - if (!cache->has_get_token) { - ret = get_kms_token_from_iam(cache, &token); - check_cmkem_ret(ret); - - set_kms_cache_auth_token(cache, token); - cmkem_free(token); - } - - *req_token = cache->req_header_token; - return CMKEM_SUCCEED; -} - -CmkemErrCode check_token_exist(size_t cache_id) -{ - CachedAuthInfo *cache = NULL; - CmkemErrCode ret = CMKEM_SUCCEED; - - ret = get_kms_cache_ptr(cache_id, &cache); - check_cmkem_ret(ret); - - if (cache->has_get_token == true) { - return CMKEM_SUCCEED; - } else { - cmkem_errmsg("unable to establish connection with kms server, because token cannot be found."); - return CMKEM_GET_TOKEN_ERR; - } -} - -CmkemErrCode refresh_cahced_token(size_t cache_id) -{ - printf("NOTICE: your IAM token has expired, we are going to update it.\n"); - - CachedAuthInfo *cache = NULL; - errno_t rc = 0; - CmkemErrCode ret = CMKEM_SUCCEED; - char *token = NULL; - - ret = get_kms_cache_ptr(cache_id, &cache); - check_cmkem_ret(ret); - - rc = memset_s(cache->req_header_token, MAX_TOKEN_BUF_LEN, 0, MAX_TOKEN_BUF_LEN); - securec_check_c(rc, "", ""); - cache->has_get_token = false; - - ret = get_kms_token_from_iam(cache, &token); - check_cmkem_ret(ret); - - set_kms_cache_auth_token(cache, token); - cmkem_free(token); - - return CMKEM_SUCCEED; -} - -CmkemErrCode catch_iam_server_err(CmkemStrList *http_res_list, HttpStatusCode http_stat_code) -{ - char *iam_resbody = NULL; - - if (http_stat_code < HTTP_BAD_REQUEST) { - return CMKEM_SUCCEED; - } else if (http_stat_code >= HTTP_INTERNAL_SERVER_ERROR) { - cmkem_errmsg("iam or porxy server error. http status code: %d.", http_stat_code); - return CMKEM_IAM_SERVER_ERR; - } - - iam_resbody = cmkem_list_val(http_res_list, -1); /* -1 == tail */ - - cmkem_errmsg("iam server error. http status code: %d, iam server error message : %s.", http_stat_code, - iam_resbody); - return CMKEM_IAM_SERVER_ERR; -} - -CmkemErrCode catch_kms_server_err(size_t cache_id, HttpReqMsg* http_req_msg, HttpConfig *http_config, - CmkemStrList **http_res_list, HttpStatusCode *http_stat_code) -{ - CmkemErrCode ret = CMKEM_SUCCEED; - char *huaweikms_resbody = cmkem_list_val(*http_res_list, 0); - - if (*http_stat_code < HTTP_BAD_REQUEST) { - return CMKEM_SUCCEED; - } else if (*http_stat_code >= HTTP_INTERNAL_SERVER_ERROR) { - cmkem_errmsg("kms or porxy server error. http status code: %d.", *http_stat_code); - return CMKEM_KMS_SERVER_ERR; - } - - ret = get_kms_err_type(huaweikms_resbody); - if (ret != CMKEM_TOKEN_EXPIRED_ERR) { - cmkem_errmsg("kms server error. http status code: %d, kms server error message : %s.", *http_stat_code, - huaweikms_resbody); - return CMKEM_KMS_SERVER_ERR; - } - - ret = refresh_cahced_token(cache_id); - check_cmkem_ret(ret); - - cmkem_list_free(*http_res_list); /* free old http response message, and try to get a new one */ - ret = http_request(http_req_msg, http_config, http_res_list, http_stat_code); - check_cmkem_ret(ret); - - if (*http_stat_code >= HTTP_BAD_REQUEST) { - cmkem_errmsg("kms server error. http status code: %d, kms server error message : %s.", *http_stat_code, - huaweikms_resbody); - cmkem_list_free(*http_res_list); - return CMKEM_KMS_SERVER_ERR; - } - - return CMKEM_SUCCEED; -} - -/* - * hex_cek = hex(raw_cek) || sha256(hex(raw_cek)) - * eg. raw_cek = "b"; - * hex_cek = "62" + "3e23e8160039594a33894f6564e1b1348bbd7a0088d42c4acb73eeaed59c009d" - * so, hex_cek_len = 2 * raw_cek_len + 32 * 2 - */ -size_t get_hexcek_len_from_rawcek_len(size_t raw_cek_len) -{ - return HEX_SIZE * raw_cek_len + SHA256_HASH_LEN * HEX_SIZE; -} - -size_t get_rawcek_len_from_hexcek_len(size_t hex_cek_len) -{ - return (hex_cek_len - HEX_SIZE * SHA256_HASH_LEN) / HEX_SIZE; -} - -size_t get_rawcek_len_from_hex_cek_cipher_len(size_t hex_cek_len) -{ - return (hex_cek_len - HEX_SIZE * SHA256_HASH_LEN - HEX_SIZE * 92) / HEX_SIZE; -} - -static CmkemErrCode get_kms_token_from_iam(CachedAuthInfo *cache, char **token) -{ - char iam_url[MAX_URL_BUF_LEN] = {0}; - const char *http_resheader[] = {"Content-Type:application/json", "charset=utf8", NULL}; - const char *res_token_tag = "X-Subject-Token"; - CmkemErrCode ret = CMKEM_SUCCEED; - - ret = are_all_kms_infos_set(cache); - check_cmkem_ret(ret); - - char *http_reqbody = get_iam_auth_req_jsontemp(cache->user_name, cache->password, cache->domain_name, - cache->project_name); - if (http_reqbody == NULL) { - return CMKEM_SET_CJSON_VALUE_ERR; - } - - get_iam_url(cache->project_name, iam_url, MAX_URL_BUF_LEN); - - HttpReqMsg http_req_msg = {HTTP_POST, iam_url, NULL, http_reqbody, http_resheader}; - HttpConfig http_config = {DEFAULT_HTTP_TIME_OUT, HTTP_MSG}; - CmkemStrList *http_res_msg_list = NULL; - HttpStatusCode http_stat_code; - - ret = http_request(&http_req_msg, &http_config, &http_res_msg_list, &http_stat_code); - cmkem_free(http_reqbody); - if (ret != CMKEM_SUCCEED) { - return ret; - } - - ret = catch_iam_server_err(http_res_msg_list, http_stat_code); - if (ret != CMKEM_SUCCEED) { - cmkem_list_free(http_res_msg_list); - return ret; - } - - *token = find_resheader(http_res_msg_list, res_token_tag); - cmkem_list_free(http_res_msg_list); - if (*token == NULL) { - cmkem_errmsg("failed to find '%s' from http response header.", res_token_tag); - return CMKEM_GET_TOKEN_ERR; - } - - return CMKEM_SUCCEED; -} - -CmkemErrCode select_cmk_entity_from_huaweikms(size_t cache_id, const char *cmk_id, char **huaweikms_resbody) -{ - CmkemErrCode ret = CMKEM_SUCCEED; - - char *req_header_token = NULL; - ret = get_kms_cache_token(cache_id, &req_header_token); - check_cmkem_ret(ret); - - char url[MAX_URL_BUF_LEN] = {0}; - ret = get_kms_url(cache_id, KMS_SELECT_CMK_REQ, url, MAX_URL_BUF_LEN); - check_cmkem_ret(ret); - - char *http_reqbody = get_select_cmk_jsontemp(cmk_id); - if (http_reqbody == NULL) { - return CMKEM_SET_CJSON_VALUE_ERR; - } - - char *contlen = make_content_length(http_reqbody); - const char *http_resheader_list[] = {"Content-Type:application/json", req_header_token, contlen, NULL}; - HttpReqMsg http_req_msg = {HTTP_POST, url, NULL, http_reqbody, http_resheader_list}; - HttpConfig http_config = {DEFAULT_HTTP_TIME_OUT, HTTP_RESBODY}; - CmkemStrList *http_resbody = NULL; - HttpStatusCode http_stat_code; - - ret = http_request(&http_req_msg, &http_config, &http_resbody, &http_stat_code); - cmkem_free(contlen); - if (ret != CMKEM_SUCCEED) { - cmkem_free(http_reqbody); - return ret; - } - - ret = catch_kms_server_err(cache_id, &http_req_msg, &http_config, &http_resbody, &http_stat_code); - cmkem_free(http_reqbody); - if (ret != CMKEM_SUCCEED) { - cmkem_list_free(http_resbody); - return ret; - } - - *huaweikms_resbody = cmkem_list_merge(http_resbody); - cmkem_list_free(http_resbody); - if (*huaweikms_resbody == NULL) { - return CMKEM_MALLOC_MEM_ERR; - } - - return CMKEM_SUCCEED; -} - -CmkemErrCode encrypt_cek_plain_by_huaweikms(size_t cache_id, CmkemStr *hex_cek_plain_join_hash, const char *cmk_id, - char **huaweikms_resbody) -{ - CmkemErrCode ret = CMKEM_SUCCEED; - - char *req_header_token = NULL; - ret = get_kms_cache_token(cache_id, &req_header_token); - check_cmkem_ret(ret); - - char url[MAX_URL_BUF_LEN] = {0}; - ret = get_kms_url(cache_id, KMS_ENC_CEK_REQ, url, MAX_URL_BUF_LEN); - check_cmkem_ret(ret); - - char *http_reqbody = get_enc_cek_plain_jsontemp(cmk_id, - hex_cek_plain_join_hash->str_val, - get_rawcek_len_from_hexcek_len(hex_cek_plain_join_hash->str_len)); - if (http_reqbody == NULL) { - return CMKEM_SET_CJSON_VALUE_ERR; - } - - char *contlen = make_content_length(http_reqbody); - const char *http_resheader_list[] = {"Content-Type:application/json", req_header_token, contlen, NULL}; - HttpReqMsg http_req_msg = {HTTP_POST, url, NULL, http_reqbody, http_resheader_list}; - HttpConfig http_config = {DEFAULT_HTTP_TIME_OUT, HTTP_RESBODY}; - CmkemStrList *http_resbody = NULL; - HttpStatusCode http_stat_code; - - ret = http_request(&http_req_msg, &http_config, &http_resbody, &http_stat_code); - cmkem_free(contlen); - if (ret != CMKEM_SUCCEED) { - cmkem_free(http_reqbody); - return ret; - } - - ret = catch_kms_server_err(cache_id, &http_req_msg, &http_config, &http_resbody, &http_stat_code); - cmkem_free(http_reqbody); - if (ret != CMKEM_SUCCEED) { - cmkem_list_free(http_resbody); - return ret; - } - - *huaweikms_resbody = cmkem_list_merge(http_resbody); - cmkem_list_free(http_resbody); - if (*huaweikms_resbody == NULL) { - return CMKEM_MALLOC_MEM_ERR; - } - - return CMKEM_SUCCEED; -} - -CmkemErrCode decrypt_cek_plain_by_huaweikms(size_t cache_id, CmkemStr *hex_cek_cipher_join_hash, const char *cmk_id, - char **huaweikms_resbody) -{ - CmkemErrCode ret = CMKEM_SUCCEED; - - char *req_header_token = NULL; - ret = get_kms_cache_token(cache_id, &req_header_token); - check_cmkem_ret(ret); - - char url[MAX_URL_BUF_LEN] = {0}; - ret = get_kms_url(cache_id, KMS_DEC_CEK_REQ, url, MAX_URL_BUF_LEN); - check_cmkem_ret(ret); - - char *http_reqbody = get_dec_cek_cipher_jsontemp(cmk_id, - hex_cek_cipher_join_hash->str_val, - get_rawcek_len_from_hex_cek_cipher_len(hex_cek_cipher_join_hash->str_len)); - if (http_reqbody == NULL) { - return CMKEM_SET_CJSON_VALUE_ERR; - } - - char *contlen = make_content_length(http_reqbody); - const char *http_resheader_list[] = {"Content-Type:application/json", req_header_token, contlen, NULL}; - HttpReqMsg http_req_msg = {HTTP_POST, url, NULL, http_reqbody, http_resheader_list}; - HttpConfig http_config = {DEFAULT_HTTP_TIME_OUT, HTTP_RESBODY}; - CmkemStrList *http_resbody = NULL; - HttpStatusCode http_stat_code; - - ret = http_request(&http_req_msg, &http_config, &http_resbody, &http_stat_code); - cmkem_free(contlen); - if (ret != CMKEM_SUCCEED) { - cmkem_free(http_reqbody); - return ret; - } - - ret = catch_kms_server_err(cache_id, &http_req_msg, &http_config, &http_resbody, &http_stat_code); - cmkem_free(http_reqbody); - if (ret != CMKEM_SUCCEED) { - cmkem_list_free(http_resbody); - return ret; - } - - *huaweikms_resbody = cmkem_list_merge(http_resbody); - cmkem_list_free(http_resbody); - - if (*huaweikms_resbody == NULL) { - return CMKEM_MALLOC_MEM_ERR; - } - - return CMKEM_SUCCEED; -} - -static CmkemErrCode check_cmk_algo_validity(const char *cmk_algo) -{ - char error_msg_buf[MAX_CMKEM_ERRMSG_BUF_SIZE] = {0}; - error_t rc = 0; - - for (size_t i = 0; supported_algorithms[i] != NULL; i++) { - if (strcasecmp(cmk_algo, supported_algorithms[i]) == 0) { - return CMKEM_SUCCEED; - } - } - - rc = sprintf_s(error_msg_buf, MAX_CMKEM_ERRMSG_BUF_SIZE, "unpported algorithm '%s', huawei kms only support: ", - cmk_algo); - securec_check_ss_c(rc, "", ""); - - for (size_t i = 0; supported_algorithms[i] != NULL; i++) { - rc = strcat_s(error_msg_buf, MAX_CMKEM_ERRMSG_BUF_SIZE, supported_algorithms[i]); - securec_check_c(rc, "\0", "\0"); - rc = strcat_s(error_msg_buf, MAX_CMKEM_ERRMSG_BUF_SIZE, " "); - securec_check_c(rc, "\0", "\0"); - } - - cmkem_errmsg(error_msg_buf); - return CMKEM_CHECK_ALGO_ERR; -} - -static CmkemErrCode check_cmk_id_validity(CmkIdentity *cmk_identity) -{ - if (strlen(cmk_identity->cmk_id_str) != strlen("00000000-0000-0000-0000-000000000000")) { - cmkem_errmsg("the length of cmk id is invalid."); - return CMKEM_CHECK_CMK_ID_ERR; - } - - return CMKEM_SUCCEED; -} - -static CmkemErrCode check_cmk_entity_validity(CmkIdentity *cmk_identity) -{ - char *huaweikms_resbody = NULL; - const char *key_list[] = {"scheduled_deletion_date", "key_state", NULL}; - CmkemStrList *value_list = NULL; - CmkemErrCode ret = CMKEM_SUCCEED; - - ret = select_cmk_entity_from_huaweikms(cmk_identity->client_cache_id, cmk_identity->cmk_id_str, &huaweikms_resbody); - check_cmkem_ret(ret); - - ret = find_value_from_http_resbody(huaweikms_resbody, key_list, &value_list); - cmkem_free(huaweikms_resbody); - check_cmkem_ret(ret); - - /* cmkem_list_val(value_list, 0) = value of key_list[0] = $scheduled_deletion_date */ - if (strlen(cmkem_list_val(value_list, 0)) > 0) { - cmkem_errmsg("cmk entity '%s' is already scheduled to be deleted, please use another cmk entity.", - cmk_identity->cmk_id_str); - cmkem_list_free(value_list); - return CMKEM_CHECK_IDENTITY_ERR; - } - - /* cmkem_list_val(value_list, 1) = value of key_list[1] = $key_state */ - if (strcmp(cmkem_list_val(value_list, 1), "2") != 0) { - cmkem_errmsg("cmk entity '%s' is unavailable.", cmk_identity->cmk_id_str); - cmkem_list_free(value_list); - return CMKEM_CHECK_IDENTITY_ERR; - } - cmkem_list_free(value_list); - - return CMKEM_SUCCEED; -} - -static CmkemErrCode check_plain_len_constant(CmkemUStr *plain) -{ - if (plain->ustr_len > 0 && plain->ustr_len % KMS_PLAIN_PACKET_LEN == 0) { - return CMKEM_SUCCEED; - } - - cmkem_errmsg("the length of the cek value must be exactly divided by %d.", KMS_PLAIN_PACKET_LEN); - return CMKEM_ENCRYPT_CEK_ERR; -} - - -static ProcessPolicy create_cmk_obj_hookfunc(CmkIdentity *cmk_identity) -{ - CmkemErrCode ret = CMKEM_SUCCEED; - - if (cmk_identity->cmk_store == NULL || strcasecmp(cmk_identity->cmk_store, "huawei_kms") != 0) { - return POLICY_CONTINUE; - } - - if (cmk_identity->cmk_id_str == NULL) { - cmkem_errmsg("failed to create client master key, failed to find arg: KEY_PATH."); - return POLICY_ERROR; - } - - if (cmk_identity->cmk_algo == NULL) { - cmkem_errmsg("failed to create client master key, failed to find arg: ALGORITHM."); - return POLICY_ERROR; - } - - ret = check_cmk_algo_validity(cmk_identity->cmk_algo); - if (ret != CMKEM_SUCCEED) { - return POLICY_ERROR; - } - - ret = check_cmk_id_validity(cmk_identity); - if (ret != CMKEM_SUCCEED) { - return POLICY_ERROR; - } - - ret = check_cmk_entity_validity(cmk_identity); - if (ret != CMKEM_SUCCEED) { - return POLICY_ERROR; - } - - return POLICY_BREAK; -} - -static ProcessPolicy encrypt_cek_plain_hookfunc(CmkemUStr *cek_plain, CmkIdentity *cmk_identity, CmkemUStr **cek_cipher) -{ - char *huaweikms_resbody = NULL; - const char *key_list[] = {"cipher_text", "datakey_length", NULL}; - CmkemStrList *value_list = NULL; - CmkemStr *hex_cek_plain_join_hash = NULL; - CmkemStr *hex_cek_cipher_join_hash = NULL; - CmkemErrCode ret = CMKEM_SUCCEED; - - if (cmk_identity->cmk_store == NULL || strcasecmp(cmk_identity->cmk_store, "huawei_kms") != 0) { - return POLICY_CONTINUE; - } - - if (check_plain_len_constant(cek_plain) != CMKEM_SUCCEED) { - return POLICY_ERROR; - } - - ret = get_hex_join_hash_from_ustr(cek_plain, &hex_cek_plain_join_hash); - if (ret != CMKEM_SUCCEED) { - return POLICY_ERROR; - } - - ret = encrypt_cek_plain_by_huaweikms(cmk_identity->client_cache_id, hex_cek_plain_join_hash, - cmk_identity->cmk_id_str, &huaweikms_resbody); - free_cmkem_str(hex_cek_plain_join_hash); - if (ret != CMKEM_SUCCEED) { - return POLICY_ERROR; - } - - ret = find_value_from_http_resbody(huaweikms_resbody, key_list, &value_list); - cmkem_free(huaweikms_resbody); - if (ret != CMKEM_SUCCEED) { - return POLICY_ERROR; - } - - hex_cek_cipher_join_hash = conv_str_to_cmkem_str(cmkem_list_val(value_list, 0)); - cmkem_list_free(value_list); - if (hex_cek_cipher_join_hash == NULL) { - return POLICY_ERROR; - } - - ret = get_ustr_from_hex_join_hash(hex_cek_cipher_join_hash, cek_cipher); - - free_cmkem_str(hex_cek_cipher_join_hash); - if (ret != CMKEM_SUCCEED) { - return POLICY_ERROR; - } - - return POLICY_BREAK; -} - -static ProcessPolicy decrypt_cek_cipher_hookfunc(CmkemUStr *cek_cipher, CmkIdentity *cmk_identity, - CmkemUStr **cek_plain) -{ - CmkemErrCode ret = CMKEM_SUCCEED; - char *huaweikms_resbody = NULL; - const char *key_list[] = {"data_key", "datakey_length", NULL}; - CmkemStrList *value_list = NULL; - CmkemStr *hex_cek_cipher_join_hash = NULL; - CmkemStr *hex_cek_plain_join_hash = NULL; - - if (cmk_identity->cmk_store == NULL || strcasecmp(cmk_identity->cmk_store, "huawei_kms") != 0) { - return POLICY_CONTINUE; - } - - ret = get_hex_join_hash_from_ustr(cek_cipher, &hex_cek_cipher_join_hash); - if (ret != CMKEM_SUCCEED) { - return POLICY_ERROR; - } - - ret = decrypt_cek_plain_by_huaweikms(cmk_identity->client_cache_id, hex_cek_cipher_join_hash, - cmk_identity->cmk_id_str, &huaweikms_resbody); - free_cmkem_str(hex_cek_cipher_join_hash); - if (ret != CMKEM_SUCCEED) { - return POLICY_ERROR; - } - - ret = find_value_from_http_resbody(huaweikms_resbody, key_list, &value_list); - cmkem_free(huaweikms_resbody); - if (ret != CMKEM_SUCCEED) { - return POLICY_ERROR; - } - - hex_cek_plain_join_hash = conv_str_to_cmkem_str(cmkem_list_val(value_list, 0)); - cmkem_list_free(value_list); - if (hex_cek_plain_join_hash == NULL) { - return POLICY_ERROR; - } - - *cek_plain = hex_to_ustr(hex_cek_plain_join_hash); - free_cmkem_str(hex_cek_plain_join_hash); - - return POLICY_BREAK; -} - -int reg_cmke_manager_huwei_kms_main() -{ - CmkEntityManager huawei_kms = { - create_cmk_obj_hookfunc, - encrypt_cek_plain_hookfunc, - decrypt_cek_cipher_hookfunc, - NULL, /* drop_cmk_obj_hook_func: no need */ - NULL, /* post_create_cmk_obj_hook_func: no need */ - }; - - return (reg_cmk_entity_manager(huawei_kms) == CMKEM_SUCCEED) ? 0 : -1; -} - -#endif /* ENABLE_HUAWEI_KMS */ diff --git a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/register_huawei_kms.h b/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/register_huawei_kms.h deleted file mode 100644 index c4c8f3c7c..000000000 --- a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/register_huawei_kms.h +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright (c) 2020 Huawei Technologies Co.,Ltd. - * - * openGauss is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * - * http://license.coscl.org.cn/MulanPSL2 - * - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - * ------------------------------------------------------------------------- - * - * register_huawei_kms.h - * Huawei KMS is an online key management service provided by Huawei Cloud. - * We support Huawei KMS to provide cmk entities for us, and we can send CEK entities - * to it for encrypting and decrypting - * - * IDENTIFICATION - * src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/register_huawei_kms.h - * - * ------------------------------------------------------------------------- - */ - -#ifndef REGISTER_HUAWEI_KMS_H -#define REGISTER_HUAWEI_KMS_H - -#include "cmkem_comm.h" -#include "cmkem_comm_http.h" - -const int USER_NAME_BUF_LEN = 24; -const int PASSWD_BUF_LEN = 24; -const int DOMAIN_NAME_BUF_LEN = 24; -const int PROJ_NAME_BUF_LEN = 24; -const int PROJ_ID_BUF_LEN = 128; -const int MAX_TOKEN_BUF_LEN = 10240; -const int DEFAULT_HTTP_TIME_OUT = 15; -const int KMS_CACHE_TBL_LEN = 1024; -const int KMS_PLAIN_PACKET_LEN = 16; - -typedef enum { - IAM_AUTH_REQ = 0, - KMS_SELECT_CMK_REQ, - KMS_ENC_CEK_REQ, - KMS_DEC_CEK_REQ, -} KmsHttpMsgType; - -extern CmkemErrCode get_kms_err_type(const char *kms_errmsg_body); -extern cJSON *get_json_temp(KmsHttpMsgType json_tree_type); -extern char *get_iam_auth_req_jsontemp(const char *user_name, const char *password, const char *domain_name, - const char *project_name); -extern char *get_select_cmk_jsontemp(const char *key_id); -extern char *get_enc_cek_plain_jsontemp(const char *cmk_id, const char *cek_plain, size_t cek_plain_len); -extern char *get_dec_cek_cipher_jsontemp(const char *cmk_id, const char *cek_cipher, size_t cek_cipher_len); -extern void get_iam_url(const char *project_name, char *url_buf, size_t buf_len); -extern CmkemErrCode get_kms_url(size_t cache_id, KmsHttpMsgType kms_msg_type, char *url_buf, size_t url_buf_len); -extern CmkemErrCode get_url_file_path(KmsHttpMsgType kms_msg_type, const char *project_id, char *url_file_path_buf, - size_t buf_len); - -extern void free_kms_cache(size_t cache_id); -extern CmkemErrCode set_kms_cache_auth_info(size_t cache_id, const char *key, const char *value); -extern char *get_kms_cache_token(size_t cache_id); -extern CmkemErrCode check_token_exist(size_t cache_id); -extern CmkemErrCode refresh_cahced_token(size_t cache_id); - -extern CmkemErrCode catch_iam_server_err(CmkemStrList *http_res_list, HttpStatusCode http_stat_code); -extern CmkemErrCode catch_kms_server_err(size_t cache_id, HttpReqMsg* http_req_msg, HttpConfig *http_config, - CmkemStrList **http_res_list, HttpStatusCode *http_stat_code); -extern size_t get_hexcek_len_from_rawcek_len(size_t raw_cek_len); -extern size_t get_rawcek_len_from_hexcek_len(size_t hex_cek_len); -extern size_t get_rawcek_len_from_hex_kms_cek_len(size_t hex_cek_len); - -extern CmkemErrCode select_cmk_entity_from_huaweikms(size_t cache_id, const char *cmk_id, char **huaweikms_resbody); -extern CmkemErrCode encrypt_cek_plain_by_huaweikms(size_t cache_id, CmkemStr *hex_cek_plain_join_hash, - const char *cmk_id, char **huaweikms_resbody); -extern CmkemErrCode decrypt_cek_plain_by_huaweikms(size_t cache_id, CmkemStr *hex_cek_cipher_join_hash, - const char *cmk_id, char **huaweikms_resbody); - -extern int reg_cmke_manager_huwei_kms_main(); - - -#endif /* REGISTER_HUAWEI_KMS_H */ diff --git a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/register_local_kms.cpp b/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/register_local_kms.cpp deleted file mode 100644 index 2af6a9dbf..000000000 --- a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/register_local_kms.cpp +++ /dev/null @@ -1,1006 +0,0 @@ -/* - * Copyright (c) 2020 Huawei Technologies Co.,Ltd. - * - * openGauss is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * - * http://license.coscl.org.cn/MulanPSL2 - * - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - * ------------------------------------------------------------------------- - * - * register_local_kms.cpp - * localkms is a lightweight key management component. - * different from other key management tools and services, localkms automatically generates CMKE when CREATE CMKO, - * it cannot manage key entities independently. - * localkms use openssl to generate cmk, then encrypt cmk plain and store cmk cipher in file. - * at the same time, we need to store/read the iv and salt that are used to derive a key to - * encrypt/decrypt cmk plain. - * - * IDENTIFICATION - * src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/register_local_kms.cpp - * - * ------------------------------------------------------------------------- - */ - -#include "cmkem_version_control.h" -#ifdef ENABLE_LOCAL_KMS - -#include "register_local_kms.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "securec.h" -#include "securec_check.h" -#include "cmkem_comm.h" -#include "aead_aes_hamc_enc_key.h" -#include "encrypt_decrypt.h" -#include "sm2_enc_key.h" -#include "cmkem_comm_algorithm.h" -#include "reg_hook_frame.h" - -const int MAX_KEY_PATH_LEN = 64; -const int MIN_KEY_PATH_LEN = 1; -const int KEY_METERIAL_LEN = 32; -const int DRIVED_KEY_LEN = 64; -const int ITERATE_ROUD = 10000; -const int KEY_FILE_HEADER_LEN = 20; - -typedef enum { - CREATE_KEY_FILE, - READ_KEY_FILE, - REMOVE_KEY_FILE -} CheckKeyFileType; - -typedef enum { - PUB_KEY_FILE, - PRIV_KEY_FILE, - PUB_KEY_ENC_IV_FILE, - PRIV_KEY_ENC_IV_FILE, -} LocalkmsFileType; - -static char g_env_value[PATH_MAX] = {0}; -static pthread_mutex_t g_env_lock; -static const char *supported_algorithms[] = {"RSA_2048", "RSA_3072", "SM2", NULL}; - -/* check the values obtained from the environment or input by the user */ -static CmkemErrCode check_env_value(const char *path_value); -static CmkemErrCode check_key_path_value(const char *key_path_value); -static CmkemErrCode check_file_exist(const char *real_file_path, CheckKeyFileType chk_type); -static CmkemErrCode check_cmk_id_validity(CmkIdentity *cmk_identity); -static CmkemErrCode check_cmk_algo_validity(const char *cmk_algo); - -/* set the environment variables needed for localkms to work */ -static char *get_env_value(const char *name); -static CmkemErrCode set_global_env_value(); - -/* - * encrypt cmk plain and write cmk plain cipher to file - * also, read cmk plain cipher from file, and decrypt it to get cmk plain - */ -static void get_file_path_from_cmk_id(const char *cmk_id_str, LocalkmsFileType file_type, char *file_path_buf, - size_t buf_len); -static CmkemErrCode create_file_and_write(const char *real_path, const unsigned char *content, size_t content_len, - bool is_write_header); -static void rm_file(const char *file_path); -static void rm_file_with_retry(const char *file_path); -static CmkemErrCode encrypt_and_write_key(const char *key_file_path, CmkemUStr *key_plain); -static CmkemErrCode write_rsa_keypair_to_file(RSA *rsa_key_pair, const char *cmk_path); -static CmkemErrCode read_content_from_file(const char *real_path, unsigned char *buf, const size_t buf_len, - size_t *content_len); -static CmkemErrCode read_cmk_plain(const char *real_cmk_path, unsigned char *cmk_plain, size_t *plain_len); -static CmkemErrCode read_and_decrypt_bio_cmk(const char *key_path, AsymmetricKeyType key_type, BIO **key_plain); -static CmkemErrCode read_rand_and_drive_key(const char *rand_path, CmkemUStr **drived_key); -static CmkemErrCode read_iv_and_salt(const char *rand_path, unsigned char *iv, size_t iv_len, unsigned char *salt, - size_t salt_len); - -static CmkemErrCode create_and_write_rsa_key_pair(const char *cmk_id, size_t rsa_key_len); -static CmkemErrCode create_and_write_sm2_key_pair(const char *cmk_id); -static CmkemErrCode write_sm2_keypair_to_file(Sm2KeyPair *sm2_key_pair, const char *cmk_path); -static CmkemErrCode encrypt_cek_with_rsa(CmkemUStr *cek_plain, const char *cmk_id_str, CmkemUStr **cek_cipher); -static CmkemErrCode decrypt_cek_with_rsa(CmkemUStr *cek_cipher, const char *cmk_id_str, CmkemUStr **cek_plain); -static CmkemErrCode encrypt_cek_with_sm2(CmkemUStr *cek_plain, const char *cmk_id_str, CmkemUStr **cek_cipher); -static CmkemErrCode decrypt_cek_with_sm2(CmkemUStr *cek_cipher, const char *cmk_id_str, CmkemUStr **cek_plain); - -/* hook functions */ -static ProcessPolicy create_cmk_obj_hookfunc(CmkIdentity *cmk_identity); -static ProcessPolicy post_create_cmk_obj_hookfunc(CmkIdentity *cmk_identity); -static ProcessPolicy encrypt_cek_plain_hookfunc(CmkemUStr *cek_plain, CmkIdentity *cmk_identity, - CmkemUStr **cek_cipher); -static ProcessPolicy decrypt_cek_cipher_hookfunc(CmkemUStr *cek_cipher, CmkIdentity *cmk_identity, - CmkemUStr **cek_plain); - -/* check the environment values input from the system */ -static CmkemErrCode check_env_value(const char *path_value) -{ - const char danger_char_list[] = {'|', ';', '&', '$', '<', '>', '`', '\\', '\'', '\"', '{', '}', '(', ')', '[',']', - '~', '*', '?', '!'}; - - for (size_t i = 0; i < strlen(path_value); i++) { - for (size_t j = 0; j < sizeof(danger_char_list); j++) { - if (path_value[i] == danger_char_list[j]) { - cmkem_errmsg("the path '%s' contains invalid character '%c'.", path_value, path_value[i]); - return CMKEM_CHECK_ENV_VAL_ERR; - } - } - } - - return CMKEM_SUCCEED; -} - -/* - * check the KEY_PATH value in SQL : - * CREATE CLIENT MASTER KEY xxx (KEY_STORE = localkms, KEY_PATH = xxx, ...) - * only support 0-9, a-z, A_Z, '_', '.', '-' - */ -static CmkemErrCode check_key_path_value(const char *key_path_value) -{ - char cur_char = 0; - const char legal_symbol[] = {'_', '.', '-'}; - - for (size_t i = 0; i < strlen(key_path_value); i++) { - cur_char = key_path_value[i]; - if (cur_char >= '0' && cur_char <= '9') { - continue; - } else if (cur_char >= 'A' && cur_char <= 'Z') { - continue; - } else if (cur_char >= 'a' && cur_char <= 'z') { - continue; - } else if (strchr(legal_symbol, cur_char) != NULL) { - continue; - } else { - cmkem_errmsg("the key_path value '%s' contains invalid charachter '%c'.", key_path_value, cur_char); - return CMKEM_CHECK_CMK_ID_ERR; - } - } - - return CMKEM_SUCCEED; -} - -/* before creating or reading cmk and rand file, we will check if the files already exist */ -static CmkemErrCode check_file_exist(const char *real_file_path, CheckKeyFileType chk_type) -{ - bool is_file_exist = false; - struct stat statbuf; - - is_file_exist = (lstat(real_file_path, &statbuf) < 0) ? false : true; - if (chk_type == CREATE_KEY_FILE && is_file_exist) { - cmkem_errmsg("cannot create file, the file '%s' already exists.\n", real_file_path); - return CMKEM_CREATE_FILE_ERR; - } else if (chk_type == READ_KEY_FILE && !is_file_exist) { - cmkem_errmsg("cannot read file, failed to find file '%s'.\n", real_file_path); - return CMKEM_FIND_FILE_ERR; - } - - return CMKEM_SUCCEED; -} - -/* - * check the $key_path_value in SQL : - * CREATE CLIENT MASTER KEY xxx (KEY_STORE = localkms, KEY_PATH = $key_path_value, ...) - * 1. is the length of $key_path_value in range (MIN_KEY_PATH_LEN, MAX_KEY_PATH_LEN) ? - * 2. does $key_path_value contain unsupport char ? - * 3. is environment variable $LOCALKMS_FILE_PATH/$GAUSSHOME set ? - * 4. are the cmk and rand keys named according $key_path_value exist ? - */ -static CmkemErrCode check_cmk_id_validity(CmkIdentity *cmk_identity) -{ - char cmk_file_path[PATH_MAX] = {0}; - CmkemErrCode ret = CMKEM_SUCCEED; - LocalkmsFileType all_file[] = {PUB_KEY_FILE, PRIV_KEY_FILE, PUB_KEY_ENC_IV_FILE, PRIV_KEY_ENC_IV_FILE}; - - if (strlen(cmk_identity->cmk_id_str) < MIN_KEY_PATH_LEN || strlen(cmk_identity->cmk_id_str) > MAX_KEY_PATH_LEN) { - return CMKEM_CHECK_CMK_ID_ERR; - } - - ret = check_key_path_value(cmk_identity->cmk_id_str); - if (ret != CMKEM_SUCCEED) { - return ret; - } - - if (strlen(g_env_value) == 0) { - ret = set_global_env_value(); - if (ret != CMKEM_SUCCEED) { - return ret; - } - } - - for (size_t i = 0; i < sizeof(all_file) / sizeof(all_file[0]); i++) { - get_file_path_from_cmk_id(cmk_identity->cmk_id_str, all_file[i], cmk_file_path, PATH_MAX); - ret = check_file_exist(cmk_file_path, CREATE_KEY_FILE); - if (ret != CMKEM_SUCCEED) { - return ret; - } - } - - return CMKEM_SUCCEED; -} - -/* - * for now, localkms only supports asymmetric cryptographic algorithm : - * 1. RSA_2048 - * 2. RSA_3072 - * 3. SM3 (for legal reasons, it is only supported in China) - */ -static CmkemErrCode check_cmk_algo_validity(const char *cmk_algo) -{ - char error_msg_buf[MAX_CMKEM_ERRMSG_BUF_SIZE] = {0}; - error_t rc = 0; - - for (size_t i = 0; supported_algorithms[i] != NULL; i++) { - if (strcasecmp(cmk_algo, supported_algorithms[i]) == 0) { - return CMKEM_SUCCEED; - } - } - - rc = sprintf_s(error_msg_buf, MAX_CMKEM_ERRMSG_BUF_SIZE, "unspported algorithm '%s', localkms only support: ", - cmk_algo); - securec_check_ss_c(rc, "", ""); - - for (size_t i = 0; supported_algorithms[i] != NULL; i++) { - rc = strcat_s(error_msg_buf, MAX_CMKEM_ERRMSG_BUF_SIZE, supported_algorithms[i]); - securec_check_c(rc, "\0", "\0"); - rc = strcat_s(error_msg_buf, MAX_CMKEM_ERRMSG_BUF_SIZE, " "); - securec_check_c(rc, "\0", "\0"); - } - - cmkem_errmsg(error_msg_buf); - return CMKEM_CHECK_ALGO_ERR; -} - -static char *get_env_value(const char *name) -{ - char *ret = NULL; - (void)pthread_mutex_lock(&g_env_lock); - ret = getenv(name); - (void)pthread_mutex_unlock(&g_env_lock); - return ret; -} - -/* - * keys generated by localkms are stored under the path ：$LOCALKMS_FILE_PATH/ - * in addition, if $LOCALKMS_FILE_PATH cannot be found, we will try to find $GAUSSHOME - */ -static CmkemErrCode set_global_env_value() -{ - char *local_kms_path = NULL; - char tmp_gausshome_buf[PATH_MAX] = {0}; - errno_t rc = 0; - bool is_get_localkms_env = false; - char tmp_env_val[PATH_MAX] = {0}; - CmkemErrCode ret = CMKEM_SUCCEED; - - local_kms_path = get_env_value("LOCALKMS_FILE_PATH"); - if (local_kms_path == NULL || realpath(local_kms_path, tmp_env_val) == NULL) { - /* fail to get LOCALKMS_FILE_PATH, then try to get GAUSSHOME */ - local_kms_path = get_env_value("GAUSSHOME"); - if (local_kms_path != NULL) { - rc = sprintf_s(tmp_gausshome_buf, sizeof(tmp_gausshome_buf), "%s/%s", local_kms_path, "/etc/localkms"); - securec_check_ss_c(rc, "", ""); - - /* judge whether the $GAUSSHOME is obtained or not */ - if (realpath(tmp_gausshome_buf, tmp_env_val) != NULL) { - is_get_localkms_env = true; - } - } - } else { - is_get_localkms_env = true; - } - - if (!is_get_localkms_env) { - cmkem_errmsg("failed to get the environment value : '%s' or the path : '%s'.", "$LOCALKMS_FILE_PATH", - "$GAUSSHOME/etc/localkms/"); - return CMKEM_GET_ENV_VAL_ERR; - } - - ret = check_env_value(tmp_env_val); - check_cmkem_ret(ret); - - rc = strcpy_s(g_env_value, PATH_MAX, tmp_env_val); - securec_check_c(rc, "", ""); - - return CMKEM_SUCCEED; -} - -/* - * in SQL : - * CREATE CLIENT MASTER KEY xxx (KEY_STORE = localkms, KEY_PATH = $key_path_value, ...) - * for each $key_path_value, we will create 4 file: - * 1. $LOCALKMS_FILE_PATH/$key_path_value.pub ： store public key cipher of asymmetric key - * 2. $LOCALKMS_FILE_PATH/$key_path_value.pub.rand ： store rands that are used to derive a key to encrypt - * public key plain - * 3. $LOCALKMS_FILE_PATH/$key_path_value.priv ： store private key cipher of asymmetric key - * 2. $LOCALKMS_FILE_PATH/$key_path_value.priv.rand ： store rands that are used to derive a key to encrypt - * private key plain - */ -static void get_file_path_from_cmk_id(const char *cmk_id_str, LocalkmsFileType file_type, char *file_path_buf, - size_t buf_len) -{ - error_t rc = 0; - const char *file_extension = NULL; - - switch (file_type) { - case PUB_KEY_FILE: - file_extension = ".pub"; - break; - case PRIV_KEY_FILE: - file_extension = ".priv"; - break; - case PUB_KEY_ENC_IV_FILE: - file_extension = ".pub.rand"; - break; - case PRIV_KEY_ENC_IV_FILE: - file_extension = ".priv.rand"; - break; - default: - break; - } - - rc = sprintf_s(file_path_buf, buf_len, "%s/%s%s", g_env_value, cmk_id_str, file_extension); - securec_check_ss_c(rc, "", ""); -} - -/* - * if para:is_write_header == true, we will create a file header, and write the size of conent to header - * after that, if trying to read all content from file, we will read header to calcule the size of content - * buffer before reading all content. - * - * although it's redundant to add a header, in order to keep the compatibility with the old version, we want - * to keep using this way - */ -static CmkemErrCode create_file_and_write(const char *real_path, const unsigned char *content, size_t content_len, - bool is_write_header) -{ - int fd = 0; - char head[KEY_FILE_HEADER_LEN] = {0}; - errno_t rc = 0; - - fd = open(real_path, O_CREAT | O_RDWR, S_IRUSR | S_IWUSR); - if (fd == -1) { - cmkem_errmsg("failed to create file '%s'.\n", real_path); - return CMKEM_CREATE_FILE_ERR; - } - - if (is_write_header) { - rc = sprintf_s(head, sizeof(head), "%lu", content_len); - securec_check_ss_c(rc, "", ""); - write(fd, head, sizeof(head)); - } - - write(fd, content, content_len); - close(fd); - - return CMKEM_SUCCEED; -} - - -static void rm_file(const char *file_path) -{ - if (remove(file_path) != 0) { -#ifndef ENABLE_UT - printf("failed to remove file '%s'.", file_path); -#endif - } -} - -static void rm_file_with_retry(const char *file_path) -{ - char tmp_path[PATH_MAX] = {0}; - - rm_file(file_path); - - if (realpath(file_path, tmp_path) != NULL) { - rm_file(file_path); - } -} - -static CmkemErrCode encrypt_and_write_key(const char *key_file_path, CmkemUStr *key_plain) -{ - unsigned char iv[KEY_METERIAL_LEN] = {0}; - unsigned char salt[KEY_METERIAL_LEN] = {0}; - unsigned char iv_salt_buf[sizeof(iv) + sizeof(salt)] = {0}; - unsigned char derived_key[DRIVED_KEY_LEN] = {0}; - unsigned char tmp_cipher[RSA2048_KEN_LEN] = {0}; - int tmp_cipher_len = 0; - char rand_file_path[PATH_MAX] = {0}; - errno_t rc = 0; - CmkemErrCode ret = CMKEM_SUCCEED; - - if (RAND_priv_bytes(iv, sizeof(iv)) != 1 || RAND_priv_bytes(salt, sizeof(salt)) != 1) { - return CMKEM_DERIVED_KEY_ERR; - } - - if (PKCS5_PBKDF2_HMAC((const char *)iv, sizeof(iv), salt, sizeof(salt), ITERATE_ROUD, EVP_sha256(), - sizeof(derived_key), derived_key) != 1) { - return CMKEM_DERIVED_KEY_ERR; - } - - AeadAesHamcEncKey derived_aead_key = AeadAesHamcEncKey(derived_key, DRIVED_KEY_LEN); - tmp_cipher_len = encrypt_data(key_plain->ustr_val, key_plain->ustr_len, derived_aead_key, - EncryptionType::DETERMINISTIC_TYPE, tmp_cipher, AEAD_AES_256_CBC_HMAC_SHA256); - if (tmp_cipher_len <= 0) { - return CMKEM_ENC_CMK_ERR; - } - - ret = create_file_and_write(key_file_path, tmp_cipher, (size_t)tmp_cipher_len, true); - if (ret != CMKEM_SUCCEED) { - return ret; - } - - rc = sprintf_s(rand_file_path, PATH_MAX, "%s.rand", key_file_path); - securec_check_ss_c(rc, "", ""); - - for (size_t i = 0; i < sizeof(iv); i++) { - iv_salt_buf[i] = iv[i]; - } - for (size_t i = 0; i < sizeof(salt); i++) { - iv_salt_buf[sizeof(iv) + i] = salt[i]; - } - - return create_file_and_write(rand_file_path, iv_salt_buf, sizeof(iv_salt_buf), true); -} - -/* - * rsa keys generated by openssl::RSA_generate_key_ex() are RSA type, - * so, we will temporarily write to the bio buffer at first, - * then, read out and stored as strings from the bio buffer - */ -static CmkemErrCode write_rsa_keypair_to_file(RSA *rsa_key_pair, const char *cmk_path) -{ - BIO *tmp_pub_key = NULL; - BIO *tmp_priv_key = NULL; - CmkemUStr *rsa_pub_key = NULL; - CmkemUStr *rsa_priv_key = NULL; - char cmk_pub_file_path[PATH_MAX] = {0}; - char cmk_priv_file_path[PATH_MAX] = {0}; - CmkemErrCode ret = CMKEM_SUCCEED; - - ret = write_rsa_keypair_to_biobuf(rsa_key_pair, &tmp_pub_key, &tmp_priv_key); - if (ret != CMKEM_SUCCEED) { - return ret; - } - - ret = read_rsa_key_from_biobuf(tmp_pub_key, &rsa_pub_key); - BIO_free(tmp_pub_key); - if (ret != CMKEM_SUCCEED) { - BIO_free(tmp_priv_key); - return ret; - } - - ret = read_rsa_key_from_biobuf(tmp_priv_key, &rsa_priv_key); - BIO_free(tmp_priv_key); - if (ret != CMKEM_SUCCEED) { - free_cmkem_ustr_with_erase(rsa_pub_key); - return ret; - } - - get_file_path_from_cmk_id(cmk_path, PUB_KEY_FILE, cmk_pub_file_path, PATH_MAX); - get_file_path_from_cmk_id(cmk_path, PRIV_KEY_FILE, cmk_priv_file_path, PATH_MAX); - - ret = encrypt_and_write_key(cmk_pub_file_path, rsa_pub_key); - free_cmkem_ustr_with_erase(rsa_pub_key); - if (ret != CMKEM_SUCCEED) { - free_cmkem_ustr_with_erase(rsa_priv_key); - return ret; - } - - ret = encrypt_and_write_key(cmk_priv_file_path, rsa_priv_key); - free_cmkem_ustr_with_erase(rsa_priv_key); - - return ret; -} - -/* the input file must contain a header which store the size value of file content */ -static CmkemErrCode read_content_from_file(const char *real_path, unsigned char *buf, const size_t buf_len, - size_t *content_len) -{ - int fd = 0; - char header[KEY_FILE_HEADER_LEN] = {0}; - - fd = open(real_path, O_RDONLY, 0); - if (fd < 0) { - cmkem_errmsg("failed to open file '%s'.\n", real_path); - return CMKEM_OPEN_FILE_ERR; - } - - if (read(fd, header, sizeof(header)) < 0) { - cmkem_errmsg("failed to read file '%s'.\n", real_path); - close(fd); - return CMKEM_READ_FILE_ERR; - } - *content_len = atoi(header); - - if (*content_len > buf_len) { - cmkem_errmsg("the header of file '%s' is invalid.\n", real_path); - close(fd); - return CMKEM_READ_FILE_ERR; - } - - if (read(fd, buf, *content_len) < 0) { - cmkem_errmsg("failed to read from file '%s'.\n", real_path); - close(fd); - return CMKEM_READ_FILE_ERR; - } - - close(fd); - return CMKEM_SUCCEED; -} - -/* - * cmk is stored as cihper, so, rather then read it directly, we will : - * 1. read rand file firstly (rand_file_path = cmk_cipher_file_path.rand) - * 2. derive a key to decrypt cmk cipher - * 3. now, we finally get cmk plain - */ -static CmkemErrCode read_cmk_plain(const char *real_cmk_path, unsigned char *cmk_plain, size_t *plain_len) -{ - char rand_file_path[PATH_MAX] = {0}; - CmkemUStr *derivied_key = NULL; - unsigned char cmk_cipher[RSA2048_KEN_LEN] = {0}; - size_t cmk_cipher_len = 0; - int tmp_plainlen = 0; - CmkemErrCode ret = CMKEM_SUCCEED; - errno_t rc = 0; - - rc = sprintf_s(rand_file_path, PATH_MAX, "%s.rand", real_cmk_path); - securec_check_ss_c(rc, "", ""); - - ret = read_rand_and_drive_key(rand_file_path, &derivied_key); - if (ret != CMKEM_SUCCEED) { - free_cmkem_ustr(derivied_key); - return ret; - } - - ret = read_content_from_file(real_cmk_path, cmk_cipher, sizeof(cmk_cipher), &cmk_cipher_len); - if (ret != CMKEM_SUCCEED) { - free_cmkem_ustr(derivied_key); - return ret; - } - - AeadAesHamcEncKey derived_aead_key = AeadAesHamcEncKey(derivied_key->ustr_val, DRIVED_KEY_LEN); - free_cmkem_ustr(derivied_key); - tmp_plainlen = decrypt_data(cmk_cipher, cmk_cipher_len, derived_aead_key, cmk_plain, AEAD_AES_256_CBC_HMAC_SHA256); - if (tmp_plainlen <= 0) { - return CMKEM_DEC_CMK_ERR; - } - *plain_len = (size_t)tmp_plainlen; - - return CMKEM_SUCCEED; -} - -static CmkemErrCode read_and_decrypt_cmk(const char *key_path, AsymmetricKeyType key_type, CmkemUStr **key_palin) -{ - CmkemErrCode ret = CMKEM_SUCCEED; - char key_file_path[PATH_MAX] = {0}; - CmkemUStr *ret_key_plain = NULL; - - if (key_type == PUBLIC_KEY) { - get_file_path_from_cmk_id(key_path, PUB_KEY_FILE, key_file_path, PATH_MAX); - } else { - get_file_path_from_cmk_id(key_path, PRIV_KEY_FILE, key_file_path, PATH_MAX); - } - - ret_key_plain = malloc_cmkem_ustr(MAX_ASYMM_KEY_BUF_LEN); - if (ret_key_plain == NULL) { - return CMKEM_MALLOC_MEM_ERR; - } - - ret = read_cmk_plain(key_file_path, ret_key_plain->ustr_val, &ret_key_plain->ustr_len); - if (ret != CMKEM_SUCCEED) { - free_cmkem_ustr_with_erase(ret_key_plain); - return ret; - } - - *key_palin = ret_key_plain; - return CMKEM_SUCCEED; -} - -static CmkemErrCode read_and_decrypt_bio_cmk(const char *key_path, AsymmetricKeyType key_type, BIO **key_plain) -{ - CmkemErrCode ret = CMKEM_SUCCEED; - int bio_write_len = 0; - CmkemUStr *tmp_key_plain = NULL; - BIO *ret_key_plain = NULL; - - ret = read_and_decrypt_cmk(key_path, key_type, &tmp_key_plain); - check_cmkem_ret(ret); - - ret_key_plain = BIO_new(BIO_s_mem()); - if (ret_key_plain == NULL) { - free_cmkem_ustr_with_erase(tmp_key_plain); - return CMKEM_MALLOC_MEM_ERR; - } - - bio_write_len = BIO_write(ret_key_plain, tmp_key_plain->ustr_val, tmp_key_plain->ustr_len); - free_cmkem_ustr_with_erase(tmp_key_plain); - if (bio_write_len < 0) { - return CMKEM_WRITE_TO_BIO_ERR; - } - - *key_plain = ret_key_plain; - return CMKEM_SUCCEED; -} - -/* - * cmk plain is encrypted byfore stored. - * the iv_and_salt are used to derive a key to encrypt cmk plain, as well as decrypt cmk cipher - */ -static CmkemErrCode read_rand_and_drive_key(const char *rand_path, CmkemUStr **drived_key) -{ - unsigned char iv[KEY_METERIAL_LEN] = {0}; - unsigned char salt[KEY_METERIAL_LEN] = {0}; - CmkemUStr *reg_drived_key = NULL; - CmkemErrCode ret = CMKEM_SUCCEED; - - ret = read_iv_and_salt(rand_path, iv, sizeof(iv), salt, sizeof(salt)); - if (ret != CMKEM_SUCCEED) { - return ret; - } - - reg_drived_key = malloc_cmkem_ustr(DRIVED_KEY_LEN); - if (reg_drived_key == NULL) { - return CMKEM_MALLOC_MEM_ERR; - } - - if (PKCS5_PBKDF2_HMAC((const char *)iv, sizeof(iv), salt, sizeof(salt), ITERATE_ROUD, EVP_sha256(), - DRIVED_KEY_LEN, reg_drived_key->ustr_val) != 1) { - free_cmkem_ustr(reg_drived_key); - return CMKEM_DERIVED_KEY_ERR; - } - - *drived_key = reg_drived_key; - return CMKEM_SUCCEED; -} - -static CmkemErrCode read_iv_and_salt(const char *rand_path, unsigned char *iv, size_t iv_len, unsigned char *salt, - size_t salt_len) -{ - unsigned char iv_salt_buf[iv_len + salt_len] = {0}; - size_t iv_salt_len = 0; - CmkemErrCode ret = CMKEM_SUCCEED; - - ret = read_content_from_file(rand_path, iv_salt_buf, sizeof(iv_salt_buf), &iv_salt_len); - if (ret != CMKEM_SUCCEED) { - return ret; - } else if (iv_salt_len < sizeof(iv_salt_buf)) { - return CMKEM_READ_FILE_ERR; - } - - for (size_t i = 0; i < iv_len; i++) { - iv[i] = iv_salt_buf[i]; - } - for (size_t i = 0; i < salt_len; i++) { - salt[i] = iv_salt_buf[iv_len + i]; - } - - return CMKEM_SUCCEED; -} - -static CmkemErrCode create_and_write_rsa_key_pair(const char *cmk_id, size_t rsa_key_len) -{ - CmkemErrCode ret = CMKEM_SUCCEED; - RSA *rsa_key_pair = NULL; - - rsa_key_pair = create_rsa_keypair(rsa_key_len); - if (rsa_key_pair == NULL) { - return CMKEM_GEN_RSA_KEY_ERR; - } - - ret = write_rsa_keypair_to_file(rsa_key_pair, cmk_id); - RSA_free(rsa_key_pair); - check_cmkem_ret(ret); - - return CMKEM_SUCCEED; -} - -static CmkemErrCode create_and_write_sm2_key_pair(const char *cmk_id) -{ - CmkemErrCode ret = CMKEM_SUCCEED; - Sm2KeyPair *sm2_key_pair = NULL; - - sm2_key_pair = generate_encrypt_pair_key(); - if (sm2_key_pair == NULL) { - return CMKEM_GEN_SM2_KEY_ERR; - } - - ret = write_sm2_keypair_to_file(sm2_key_pair, cmk_id); - check_cmkem_ret(ret); - - return CMKEM_SUCCEED; -} - -static CmkemErrCode write_sm2_keypair_to_file(Sm2KeyPair *sm2_key_pair, const char *cmk_path) -{ - char cmk_pub_file_path[PATH_MAX] = {0}; - char cmk_priv_file_path[PATH_MAX] = {0}; - CmkemErrCode ret = CMKEM_SUCCEED; - - get_file_path_from_cmk_id(cmk_path, PUB_KEY_FILE, cmk_pub_file_path, PATH_MAX); - get_file_path_from_cmk_id(cmk_path, PRIV_KEY_FILE, cmk_priv_file_path, PATH_MAX); - - ret = encrypt_and_write_key(cmk_pub_file_path, sm2_key_pair->pub_key); - check_cmkem_ret(ret); - - ret = encrypt_and_write_key(cmk_priv_file_path, sm2_key_pair->priv_key); - check_cmkem_ret(ret); - - return CMKEM_SUCCEED; -} - -static CmkemErrCode encrypt_cek_with_rsa(CmkemUStr *cek_plain, const char *cmk_id_str, CmkemUStr **cek_cipher) -{ - CmkemErrCode ret = CMKEM_SUCCEED; - BIO *cmk_plain = NULL; - RSA *rsa_cmk_plain = NULL; - CmkemUStr *ret_cek_cipher = NULL; - int enc_ret = 0; - - ret = read_and_decrypt_bio_cmk(cmk_id_str, PUBLIC_KEY, &cmk_plain); - check_cmkem_ret(ret); - - rsa_cmk_plain = PEM_read_bio_RSA_PUBKEY(cmk_plain, NULL, NULL, NULL); - BIO_free(cmk_plain); - if (rsa_cmk_plain == NULL) { - return CMKEM_READ_FROM_BIO_ERR; - } - - ret_cek_cipher = malloc_cmkem_ustr(MAX_ASYMM_KEY_BUF_LEN); - if (ret_cek_cipher == NULL) { - RSA_free(rsa_cmk_plain); - return CMKEM_MALLOC_MEM_ERR; - } - - enc_ret = RSA_public_encrypt(cek_plain->ustr_len, cek_plain->ustr_val, ret_cek_cipher->ustr_val, rsa_cmk_plain, - RSA_PKCS1_OAEP_PADDING); - RSA_free(rsa_cmk_plain); - if (enc_ret == -1) { - free_cmkem_ustr(ret_cek_cipher); - return CMKEM_RSA_ENCRYPT_ERR; - } - - ret_cek_cipher->ustr_len = (size_t)enc_ret; - *cek_cipher = ret_cek_cipher; - - return CMKEM_SUCCEED; -} - -static CmkemErrCode decrypt_cek_with_rsa(CmkemUStr *cek_cipher, const char *cmk_id_str, CmkemUStr **cek_plain) -{ - CmkemErrCode ret = CMKEM_SUCCEED; - BIO *cmk_plain = NULL; - RSA *rsa_cmk_plain = NULL; - CmkemUStr *ret_cek_plain = NULL; - int enc_ret = 0; - - ret = read_and_decrypt_bio_cmk(cmk_id_str, PRIVATE_KEY, &cmk_plain); - check_cmkem_ret(ret); - - rsa_cmk_plain = PEM_read_bio_RSAPrivateKey(cmk_plain, NULL, NULL, NULL); - BIO_free(cmk_plain); - if (rsa_cmk_plain == NULL) { - return CMKEM_READ_FROM_BIO_ERR; - } - - ret_cek_plain = malloc_cmkem_ustr(MAX_ASYMM_KEY_BUF_LEN); - if (ret_cek_plain == NULL) { - RSA_free(rsa_cmk_plain); - return CMKEM_MALLOC_MEM_ERR; - } - - enc_ret = RSA_private_decrypt(cek_cipher->ustr_len, cek_cipher->ustr_val, ret_cek_plain->ustr_val, rsa_cmk_plain, - RSA_PKCS1_OAEP_PADDING); - RSA_free(rsa_cmk_plain); - if (enc_ret == -1) { - free_cmkem_ustr_with_erase(ret_cek_plain); - return CMKEM_RSA_DECRYPT_ERR; - } - - ret_cek_plain->ustr_len = (size_t) enc_ret; - *cek_plain = ret_cek_plain; - - return CMKEM_SUCCEED; -} - -static CmkemErrCode encrypt_cek_with_sm2(CmkemUStr *cek_plain, const char *cmk_id_str, CmkemUStr **cek_cipher) -{ - CmkemErrCode ret = CMKEM_SUCCEED; - CmkemUStr *cmk_plain = NULL; - - ret = read_and_decrypt_cmk(cmk_id_str, PUBLIC_KEY, &cmk_plain); - check_cmkem_ret(ret); - - ret = encrypt_with_sm2_pubkey(cek_plain, cmk_plain, cek_cipher); - free_cmkem_ustr_with_erase(cmk_plain); - check_cmkem_ret(ret); - - return CMKEM_SUCCEED; -} - -static CmkemErrCode decrypt_cek_with_sm2(CmkemUStr *cek_cipher, const char *cmk_id_str, CmkemUStr **cek_plain) -{ - CmkemErrCode ret = CMKEM_SUCCEED; - CmkemUStr *cmk_plain = NULL; - - ret = read_and_decrypt_cmk(cmk_id_str, PRIVATE_KEY, &cmk_plain); - check_cmkem_ret(ret); - - ret = decrypt_with_sm2_privkey(cek_cipher, cmk_plain, cek_plain); - free_cmkem_ustr_with_erase(cmk_plain); - check_cmkem_ret(ret); - - return CMKEM_SUCCEED; -} - -static ProcessPolicy create_cmk_obj_hookfunc(CmkIdentity *cmk_identity) -{ - CmkemErrCode ret = CMKEM_SUCCEED; - - if (cmk_identity->cmk_store == NULL || strcasecmp(cmk_identity->cmk_store, "localkms") != 0) { - return POLICY_CONTINUE; - } - - if (cmk_identity->cmk_id_str == NULL) { - cmkem_errmsg("failed to create client master key, failed to find arg: KEY_PATH."); - return POLICY_ERROR; - } - - if (cmk_identity->cmk_algo == NULL) { - cmkem_errmsg("failed to create client master key, failed to find arg: ALGORITHM."); - return POLICY_ERROR; - } - - ret = check_cmk_id_validity(cmk_identity); - if (ret != CMKEM_SUCCEED) { - return POLICY_ERROR; - } - - ret = check_cmk_algo_validity(cmk_identity->cmk_algo); - if (ret != CMKEM_SUCCEED) { - return POLICY_ERROR; - } - - return POLICY_BREAK; -} - -static ProcessPolicy post_create_cmk_obj_hookfunc(CmkIdentity *cmk_identity) -{ - CmkemErrCode ret = CMKEM_SUCCEED; - - if (cmk_identity->cmk_store == NULL || strcasecmp(cmk_identity->cmk_store, "localkms") != 0) { - return POLICY_CONTINUE; - } - - switch (get_algo_by_str(cmk_identity->cmk_algo)) { - case RSA_2048: - ret = create_and_write_rsa_key_pair(cmk_identity->cmk_id_str, RSA2048_KEN_LEN); - break; - case RSA_3072: - ret = create_and_write_rsa_key_pair(cmk_identity->cmk_id_str, RSA3072_KEN_LEN); - break; - case SM2: - ret = create_and_write_sm2_key_pair(cmk_identity->cmk_id_str); - default: - break; - } - - if (ret != CMKEM_SUCCEED) { - return POLICY_ERROR; - } - - return PLLICY_PROCESSED; -} - -static ProcessPolicy encrypt_cek_plain_hookfunc(CmkemUStr *cek_plain, CmkIdentity *cmk_identity, CmkemUStr **cek_cipher) -{ - CmkemErrCode ret = CMKEM_SUCCEED; - - if (cmk_identity->cmk_store == NULL || strcasecmp(cmk_identity->cmk_store, "localkms") != 0) { - return POLICY_CONTINUE; - } - - switch (get_algo_by_str(cmk_identity->cmk_algo)) { - case RSA_2048: - case RSA_3072: - ret = encrypt_cek_with_rsa(cek_plain, cmk_identity->cmk_id_str, cek_cipher); - break; - case SM2: - ret = encrypt_cek_with_sm2(cek_plain, cmk_identity->cmk_id_str, cek_cipher); - default: - break; - } - - if (ret != CMKEM_SUCCEED) { - return POLICY_ERROR; - } - - return POLICY_BREAK; -} - -static ProcessPolicy decrypt_cek_cipher_hookfunc(CmkemUStr *cek_cipher, CmkIdentity *cmk_identity, - CmkemUStr **cek_plain) -{ - CmkemErrCode ret = CMKEM_SUCCEED; - - if (cmk_identity->cmk_store == NULL || strcasecmp(cmk_identity->cmk_store, "localkms") != 0) { - return POLICY_CONTINUE; - } - - switch (get_algo_by_str(cmk_identity->cmk_algo)) { - case RSA_2048: - case RSA_3072: - ret = decrypt_cek_with_rsa(cek_cipher, cmk_identity->cmk_id_str, cek_plain); - break; - case SM2: - ret = decrypt_cek_with_sm2(cek_cipher, cmk_identity->cmk_id_str, cek_plain); - default: - break; - } - - if (ret != CMKEM_SUCCEED) { - return POLICY_ERROR; - } - - return POLICY_BREAK; -} - -static ProcessPolicy drop_cmk_obj_hookfunc(CmkIdentity *cmk_identity) -{ - char pub_cmk_file[PATH_MAX] = {0}; - char priv_cmk_file[PATH_MAX] = {0}; - errno_t rc = 0; - - if (cmk_identity->cmk_store == NULL || strcasecmp(cmk_identity->cmk_store, "localkms") != 0) { - return POLICY_CONTINUE; - } - - get_file_path_from_cmk_id(cmk_identity->cmk_id_str, PUB_KEY_FILE, pub_cmk_file, PATH_MAX); - get_file_path_from_cmk_id(cmk_identity->cmk_id_str, PRIV_KEY_FILE, priv_cmk_file, PATH_MAX); - - rm_file_with_retry(pub_cmk_file); - rm_file_with_retry(priv_cmk_file); - - rc = strcat_s(pub_cmk_file, PATH_MAX, ".rand"); - securec_check_c(rc, "", ""); - rc = strcat_s(priv_cmk_file, PATH_MAX, ".rand"); - securec_check_c(rc, "", ""); - - rm_file_with_retry(pub_cmk_file); - rm_file_with_retry(priv_cmk_file); - - return POLICY_BREAK; -} - -int reg_cmke_manager_local_kms_main() -{ - if (strlen(g_env_value) == 0) { - CmkemErrCode ret = set_global_env_value(); - if (ret != CMKEM_SUCCEED) { - return -1; - } - } - - CmkEntityManager localkms = { - create_cmk_obj_hookfunc, - encrypt_cek_plain_hookfunc, - decrypt_cek_cipher_hookfunc, - drop_cmk_obj_hookfunc, - post_create_cmk_obj_hookfunc, - }; - - return (reg_cmk_entity_manager(localkms) == CMKEM_SUCCEED) ? 0 : -1; -} - -#endif /* ENABLE_LOCAL_KMS */ diff --git a/src/common/interfaces/libpq/client_logic_hooks/encryption_hooks/encrypt_decrypt.cpp b/src/common/interfaces/libpq/client_logic_hooks/encryption_hooks/encrypt_decrypt.cpp deleted file mode 100755 index 4ee598d20..000000000 --- a/src/common/interfaces/libpq/client_logic_hooks/encryption_hooks/encrypt_decrypt.cpp +++ /dev/null @@ -1,522 +0,0 @@ -/* - * Copyright (c) 2020 Huawei Technologies Co.,Ltd. - * - * openGauss is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * - * http://license.coscl.org.cn/MulanPSL2 - * - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - * ------------------------------------------------------------------------- - * - * encrypt_decrypt.cpp - * - * IDENTIFICATION - * src\common\interfaces\libpq\client_logic_hooks\encryption_hooks\encrypt_decrypt.cpp - * - * ------------------------------------------------------------------------- - */ - -#include -#include - -#include - -#include -#include -#include "encrypt_decrypt.h" - -/* Key size in bytes */ -static const int g_key_size = 32; - -static const int g_auth_tag_size = 32; - -/* Block size in bytes. AES uses 16 byte blocks. */ -static const int g_block_size = 16; - -static const int g_iv_size = 16; /* 128 bit */ - -static const int g_algo_version_size = 4; - -/* - * Minimum Length of cipher_text. This value is 4 (version byte) + 32 (authentication tag) + 16 (IV) + 16 (minimum of 1 - * block of cipher Text) - */ -static const int min_ciph_len_in_bytes_with_authen_tag = - g_algo_version_size + g_iv_size + g_block_size + g_auth_tag_size; - -static const unsigned char algo_version[4] = {'1'}; - -static int encrypt(const unsigned char *plaintext, int plaintext_len, const unsigned char *key, const unsigned char *iv, - unsigned char *ciphertext, const EVP_CIPHER *cipher); -static int decrypt(const unsigned char *ciphertext, int ciphertext_len, const unsigned char *key, - const unsigned char *iv, unsigned char *plaintext, const EVP_CIPHER *cipher); -static int my_memcmp(const void *buffer1, const void *buffer2, int count); -static bool hmac_sha256_iv(AeadAesHamcEncKey &column_encryption_key, int keylen, const unsigned char *data, - int datalen, unsigned char *result); -static bool hmac_sha256_mac(AeadAesHamcEncKey &column_encryption_key, int keylen, const unsigned char *data, - int datalen, unsigned char *result); -static bool sm3(const unsigned char *data, int datalen, unsigned char *result); -static bool get_hash_by_cek_algo(ColumnEncryptionAlgorithm column_encryption_algorithm, int hmac_length, - AeadAesHamcEncKey &column_encryption_key, const unsigned char *data, unsigned char *authentication_tag); - -/* - * @Brief : bool cached_hmac() - * @Description : computes hmac of a given data block. Calls init, update, and final. - * when ctx_ptr_template is passed in, this function uses that context, and algo type and keys are ignored - * saves time for context init - * when ctx_ptr is passed in, the function saves time for new/free - * @return : success:true, failed:false. - */ -bool cached_hmac(unsigned long algo_type, const unsigned char *key, int key_len, const unsigned char *data, - int data_len, unsigned char *result, unsigned int *result_len, HmacCtxGroup *cached_ctx_group) -{ - static unsigned char result_container[EVP_MAX_MD_SIZE]; - static const unsigned char dummy_key[1] = {'\0'}; - if (cached_ctx_group == NULL) { - return false; - } - /* reset or new a worker ctx */ - if (cached_ctx_group->ctx_worker != NULL) { - HMAC_CTX_reset(cached_ctx_group->ctx_worker); - } else { - cached_ctx_group->ctx_worker = HMAC_CTX_new(); - if (cached_ctx_group->ctx_worker == NULL) { - goto err; - } - } - /* create a new template ctx */ - if (cached_ctx_group->ctx_template == NULL) { - /* making new template in local_ctx_group */ - const EVP_MD* evp_md = get_evp_md_by_id(algo_type); - cached_ctx_group->ctx_template = HMAC_CTX_new(); - if (cached_ctx_group->ctx_template == NULL) { - goto err; - } - if (key == NULL && key_len == 0) { - key = dummy_key; - } - if (!HMAC_Init_ex(cached_ctx_group->ctx_template, key, key_len, evp_md, NULL)) { - goto err; - } - } - /* copy contents of template to worker ctx */ - if (!HMAC_CTX_copy(cached_ctx_group->ctx_worker, cached_ctx_group->ctx_template)) { - goto err; - } - - /* run std lib func to calculate hmac */ - if (result == NULL) { - result = result_container; - } - if (!HMAC_Update(cached_ctx_group->ctx_worker, data, data_len)) { - goto err; - } - if (!HMAC_Final(cached_ctx_group->ctx_worker, result, result_len)) { - goto err; - } - return true; -err: - cached_ctx_group->free_hmac_ctx_all(); - return false; -} - -static const EVP_CIPHER* get_evp_cipher_md_by_algo(ColumnEncryptionAlgorithm columnEncryptionAlgorithm) -{ - const EVP_CIPHER *cipher = NULL; - switch (columnEncryptionAlgorithm) { - case ColumnEncryptionAlgorithm::AEAD_AES_256_CBC_HMAC_SHA256: - cipher = EVP_aes_256_cbc(); - break; - case ColumnEncryptionAlgorithm::AEAD_AES_128_CBC_HMAC_SHA256: - cipher = EVP_aes_128_cbc(); - break; - case ColumnEncryptionAlgorithm::SM4_SM3: - cipher = EVP_sm4_cbc(); - break; - default: - break; - } - return cipher; -} - -/* - * Computes a keyed hash of a given text - * currently used for both generating a MAC and as a KDF - */ -static bool hmac_sha256_iv(AeadAesHamcEncKey &column_encryption_key, int keylen, const unsigned char *data, - int datalen, unsigned char *result) -{ - unsigned int result_len = g_key_size; - if (!cached_hmac(NID_hmacWithSHA256, (const GS_UCHAR *)column_encryption_key.get_iv_key(), keylen, data, - datalen, result, &result_len, &column_encryption_key.hmac_ctx_group_iv)) { - return false; - } - return true; -} - -static bool hmac_sha256_mac(AeadAesHamcEncKey &column_encryption_key, int keylen, const unsigned char *data, - int datalen, unsigned char *result) -{ - unsigned int result_len = g_key_size; - if (!cached_hmac(NID_hmacWithSHA256, (const GS_UCHAR *)column_encryption_key.get_mac_key(), keylen, data, - datalen, result, &result_len, &column_encryption_key.hmac_ctx_group_mac)) { - return false; - } - return true; -} - -/* - * Computes a keyed hash of a given text by sm3 - */ -static bool sm3(const unsigned char *data, int datalen, unsigned char *result) -{ - unsigned int result_len = g_key_size; - EVP_MD_CTX *md_ctx = NULL; - md_ctx = EVP_MD_CTX_new(); - if (md_ctx == NULL) { - printf("ERROR(CLIENT): Fail to create the context in sm3 algorithm.\n"); - return false; - } - if (!EVP_DigestInit_ex(md_ctx, EVP_sm3(), NULL)) { - printf("ERROR(CLIENT): Fail to initialise the context in sm3 algorithm.\n"); - EVP_MD_CTX_free(md_ctx); - return false; - } - if (!EVP_DigestUpdate(md_ctx, data, (size_t)datalen)) { - printf("ERROR(CLIENT): Fail to compute digest in sm3 algorithm.\n"); - EVP_MD_CTX_free(md_ctx); - return false; - } - if (!EVP_DigestFinal_ex(md_ctx, result, &result_len)) { - printf("ERROR(CLIENT): Fail to compute digest final in sm3 algorithm.\n"); - EVP_MD_CTX_free(md_ctx); - return false; - } - EVP_MD_CTX_free(md_ctx); - return true; -} - -static bool get_hash_by_cek_algo(ColumnEncryptionAlgorithm column_encryption_algorithm, int hmac_length, - AeadAesHamcEncKey &column_encryption_key, const unsigned char *data, unsigned char *authentication_tag) -{ - bool res = false; - if (column_encryption_algorithm == ColumnEncryptionAlgorithm::SM4_SM3) { - res = sm3(data, hmac_length, authentication_tag); - } else { - res = hmac_sha256_mac(column_encryption_key, g_auth_tag_size, data, hmac_length, authentication_tag); - } - return res; -} - -/* - * To calculate the ciphertext buffer size - */ -int get_cipher_text_size(int plain_text_size) -{ - int numBlocks = plain_text_size / g_block_size + 1; - int cipher_len = numBlocks * g_block_size; - /* Output buffer size = size of VersionByte + Authentication Tag + IV + cipher Text blocks. */ - return (g_algo_version_size + g_auth_tag_size + g_iv_size + cipher_len); -} - - -/* - * Encryption data - */ -int encrypt_data(const unsigned char *plain_text, int plain_text_length, AeadAesHamcEncKey &column_encryption_key, - EncryptionType encryption_type, unsigned char *result, ColumnEncryptionAlgorithm column_encryption_algorithm) -{ - if (plain_text == NULL || plain_text_length <= 0 || encryption_type == EncryptionType::INVALID_TYPE || - result == NULL) { - /* invalid input */ - return 0; - } - errno_t res = EOK; - /* Prepare IV.IV should be 1 single block (16 bytes) */ - unsigned char _iv [g_key_size + 1] = {0}; - unsigned char iv_truncated[g_iv_size + 1] = {0}; - if (encryption_type == EncryptionType::DETERMINISTIC_TYPE) { - /* - * determenistic encryption - we create an initiailization vector based on the plaintext - to make the - * encryption CPA-secure - * HMAC_SHA_256 - */ - hmac_sha256_iv(column_encryption_key, g_auth_tag_size, plain_text, plain_text_length, _iv); - - /* iv is truncated to 128 bits. */ - res = memcpy_s(iv_truncated, g_iv_size + 1, _iv, g_block_size); - if (res != EOK) { - securec_check_c(res, "\0", "\0"); - printf("ERROR(CLIENT): fail to copy 128 bit iv from 256 bit iv value.\n"); - return 0; - } - } else { - if (encryption_type != EncryptionType::RANDOMIZED_TYPE) { - return 0; - } - - if (RAND_priv_bytes(iv_truncated, g_block_size) != 1) { - return 0; - } - } - const EVP_CIPHER *cipher = get_evp_cipher_md_by_algo(column_encryption_algorithm); - if (cipher == NULL) { - printf("ERROR(CLIENT): invalid column encryption encryption algorithm, please check it!.\n"); - return 0; - } - const unsigned char *key = column_encryption_key.get_encyption_key(); - /* Add the ciphertext */ - int cipherStart = g_algo_version_size + g_auth_tag_size + g_iv_size; - int cipherTextSize = 0; - if (column_encryption_algorithm == ColumnEncryptionAlgorithm::AEAD_AES_128_CBC_HMAC_SHA256) { - /* iv is truncated to 128 bits. */ - int encrypt_key_len = g_key_size / 2 + 1; - unsigned char encrypt_key[encrypt_key_len] = {0}; - res = memcpy_s(encrypt_key, encrypt_key_len, key, g_key_size / 2); - if (res != EOK) { - printf("ERROR(CLIENT): Fail to copy 128 bit from 256 bit key value.\n"); - securec_check_c(res, "\0", "\0"); - return 0; - } - cipherTextSize = encrypt(plain_text, plain_text_length, encrypt_key, iv_truncated, - result + cipherStart, cipher); - } else { - cipherTextSize = encrypt(plain_text, plain_text_length, key, iv_truncated, result + cipherStart, cipher); - } - if (cipherTextSize < 0) { - /* failed to encrypt */ - return 0; - } - - /* add the Algorithm Version */ - res = memcpy_s(result + g_auth_tag_size, g_algo_version_size, algo_version, g_algo_version_size); - securec_check_c(res, "\0", "\0"); - - /* add the IV */ - int ivStartIndex = g_auth_tag_size + g_algo_version_size; - res = memcpy_s(result + ivStartIndex, g_iv_size, iv_truncated, g_iv_size); - securec_check_c(res, "\0", "\0"); - - /* add the HMAC (of versionbyte + IV + Ciphertext) */ - int hmacDataSize = g_algo_version_size + g_iv_size + cipherTextSize; - bool hmac_res = get_hash_by_cek_algo(column_encryption_algorithm, hmacDataSize, column_encryption_key, - result + g_auth_tag_size, result); - if (!hmac_res) { - printf("ERROR(CLIENT): Fail to compute a keyed hash of a given text.\n"); - return 0; - } - - return (g_auth_tag_size + hmacDataSize); -} - -/* - * Decryption steps - * 1. Validate version byte - * 2. Validate Authentication tag - * 3. Decrypt the message - */ -int decrypt_data(const unsigned char *cipher_text, int cipher_text_length, - AeadAesHamcEncKey &column_encryption_key, unsigned char *decryptedtext, - ColumnEncryptionAlgorithm column_encryption_algorithm) -{ - if (cipher_text == NULL || cipher_text_length <= 0 || decryptedtext == NULL) { - return 0; - } - - if (cipher_text_length < min_ciph_len_in_bytes_with_authen_tag) { - printf("ERROR(CLIENT): The length of cipher_text is invalid, cannot decrypt.\n"); - return 0; - } - - if (cipher_text[g_auth_tag_size] != '1') { - /* Cipher text was computed with a different algorithm version than this. */ - printf("ERROR(CLIENT): Version byte of cipher_text is invalid, cannot decrypt.\n"); - return 0; - } - unsigned char iv [g_iv_size] = {0}; - - errno_t rc = memcpy_s(iv, g_iv_size, cipher_text + g_auth_tag_size + g_algo_version_size, g_iv_size); - securec_check_c(rc, "\0", "\0"); - - /* Computing the authentication tag */ - unsigned char authenticationTag [g_auth_tag_size] = {0}; - int HMAC_length = cipher_text_length - g_auth_tag_size; - bool res = get_hash_by_cek_algo(column_encryption_algorithm, HMAC_length, column_encryption_key, - cipher_text + g_auth_tag_size, authenticationTag); - if (!res) { - printf("ERROR(CLIENT): Fail to compute a keyed hash of a given text.\n"); - return 0; - } - - int cmp_result = my_memcmp(authenticationTag, cipher_text, g_auth_tag_size); - if (cmp_result != 0) { - /* MAC check failed */ - return 0; - } - - /* Decrypt the ciphertext */ - int cipher_start_index = g_auth_tag_size + g_algo_version_size + g_iv_size; // this is where cipher starts. - int cipher_value_length = cipher_text_length - cipher_start_index; - const EVP_CIPHER *cipher = get_evp_cipher_md_by_algo(column_encryption_algorithm); - if (cipher == NULL) { - printf("ERROR(CLIENT): invalid column encryption encryption algorithm, please check it!.\n"); - return 0; - } - const unsigned char *key = column_encryption_key.get_encyption_key(); - int decryptedtext_len = 0; - if (column_encryption_algorithm == ColumnEncryptionAlgorithm::AEAD_AES_128_CBC_HMAC_SHA256) { - /* iv is truncated to 128 bits. */ - int encrypt_key_len = g_key_size / 2 + 1; - unsigned char encrypt_key[encrypt_key_len] = {0}; - errno_t result = memcpy_s(encrypt_key, encrypt_key_len, key, g_key_size / 2); - if (result != EOK) { - printf("ERROR(CLIENT): Fail to copy 128 bit from 256 bit key value.\n"); - securec_check_c(result, "\0", "\0"); - return 0; - } - decryptedtext_len = decrypt(cipher_text + cipher_start_index, cipher_value_length, - encrypt_key, iv, decryptedtext, cipher); - } else { - decryptedtext_len = decrypt(cipher_text + cipher_start_index, cipher_value_length, - key, iv, decryptedtext, cipher); - } - if (decryptedtext_len < 0) { - return 0; - } - - /* Add a NULL terminator. We are expecting printable text */ - decryptedtext[decryptedtext_len] = '\0'; - return decryptedtext_len; -} - -/* - * encrypt plaintext thought AES_256_cbc algorithm - * cell_ciphertext = AES-CBC-256(enc_key, cell_iv, cell_data) with PKCS7 padding. - */ -static int encrypt(const unsigned char *plaintext, int plaintext_len, const unsigned char *key, const unsigned char *iv, - unsigned char *ciphertext, const EVP_CIPHER *cipher) -{ - if (plaintext == NULL || plaintext_len <= 0 || key == NULL || iv == NULL || ciphertext == NULL) { - return 0; - } - - /* Create and initialise the context */ - EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new(); - if (ctx == NULL) { - printf("ERROR(CLIENT): Fail to create and initialise the context.\n"); - return -1; - } - /* - * Initialise the encryption operation. IMPORTANT - ensure you use a key - * and IV size appropriate for your cipher - * In this example we are using 256 bit AES (i.e. a 256 bit key). The - * IV size for *most* modes is the same as the block size. For AES this - * is 128 bits - */ - - if (EVP_EncryptInit_ex(ctx, cipher, NULL, key, iv) != 1) { - printf("ERROR(CLIENT): Fail to create new cipher.\n"); - EVP_CIPHER_CTX_free(ctx); - return -1; - } - /* - * Provide the message to be encrypted, and obtain the encrypted output. - * EVP_EncryptUpdate can be called multiple times if necessary - */ - int len = 0; - int ciphertext_len = 0; - if (EVP_EncryptUpdate(ctx, ciphertext, &len, plaintext, plaintext_len) != 1) { - printf("ERROR(CLIENT): Fail to encrypt.\n"); - EVP_CIPHER_CTX_free(ctx); - return -1; - } - ciphertext_len = len; - /* - * Finalise the encryption. Further ciphertext bytes may be written at - * this stage. - */ - if (EVP_EncryptFinal_ex(ctx, ciphertext + len, &len) != 1) { - printf("ERROR(CLIENT): Fail to encrypt final.\n"); - EVP_CIPHER_CTX_free(ctx); - return -1; - } - ciphertext_len += len; - /* Clean up */ - EVP_CIPHER_CTX_free(ctx); - - return ciphertext_len; -} - -/* - * Decrypt the message though aes_256_cbc algorithm - */ -static int decrypt(const unsigned char *ciphertext, int ciphertext_len, const unsigned char *key, - const unsigned char *iv, unsigned char *plaintext, const EVP_CIPHER *cipher) -{ - EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new(); - /* Create and initialise the context */ - if (ctx == NULL) { - printf("ERROR(CLIENT): keymanagecannot new ctx.\n"); - return -1; - } - /* - * Initialise the decryption operation. IMPORTANT - ensure you use a key - * and IV size appropriate for your cipher - * In this example we are using 256 bit AES (i.e. a 256 bit key). The - * IV size for *most* modes is the same as the block size. For AES this - * is 128 bits - */ - - if (EVP_DecryptInit_ex(ctx, cipher, NULL, key, iv) != 1) { - printf("ERROR(CLIENT): cannot create new cipher.\n"); - EVP_CIPHER_CTX_free(ctx); - return -1; - } - - /* - * Provide the message to be decrypted, and obtain the plaintext output. - * EVP_DecryptUpdate can be called multiple times if necessary. - */ - int len = 0; - int plaintext_len = 0; - if (EVP_DecryptUpdate(ctx, plaintext, &len, ciphertext, ciphertext_len) != 1) { - printf("ERROR(CLIENT): cannot EVP_EncryptUpdate.\n"); - EVP_CIPHER_CTX_free(ctx); - return -1; - } - plaintext_len = len; - /* - * Finalise the decryption. Further plaintext bytes may be written at - * this stage. - */ - if (EVP_DecryptFinal_ex(ctx, plaintext + len, &len) != 1) { - printf("ERROR(CLIENT): cannot EVP_EncryptFinal_ex.\n"); - EVP_CIPHER_CTX_free(ctx); - return -1; - } - plaintext_len += len; - /* Clean up */ - EVP_CIPHER_CTX_free(ctx); - - return plaintext_len; -} - -static int my_memcmp(const void *buffer1, const void *buffer2, int count) -{ - if (!count) { - return 0; - } - while (count--) { - if (*(char *)buffer1 != *(char *)buffer2) { - return (*((unsigned char *)buffer1) - *((unsigned char *)buffer2)); - } - buffer1 = (char *)buffer1 + 1; - buffer2 = (char *)buffer2 + 1; - } - return 0; -} diff --git a/src/common/interfaces/libpq/client_logic_hooks/encryption_hooks/encryption_column_hook_executor.cpp b/src/common/interfaces/libpq/client_logic_hooks/encryption_hooks/encryption_column_hook_executor.cpp index 198901aba..c6011221c 100644 --- a/src/common/interfaces/libpq/client_logic_hooks/encryption_hooks/encryption_column_hook_executor.cpp +++ b/src/common/interfaces/libpq/client_logic_hooks/encryption_hooks/encryption_column_hook_executor.cpp @@ -37,8 +37,7 @@ #include "libpq-int.h" #include "pqexpbuffer.h" #include "catalog/pg_type.h" -#include "cmk_entity_manager_hooks/reg_cmkem_manager_main.h" -#include "cmk_entity_manager_hooks/reg_hook_frame.h" +#include "keymgr/security_key_adpt.h" bool is_cmk_algo_sm(CmkAlgorithm cmk_algo) { @@ -131,8 +130,8 @@ int EncryptionColumnHookExecutor::process_data_impl(const ICachedColumn *cached_ } else if (data_type == BYTEAWITHOUTORDERCOLOID) { encryption_type = EncryptionType::RANDOMIZED_TYPE; } - int encryptedSize = encrypt_data(data, data_size, aesCbcEncryptionKey, encryption_type, processed_data, - column_encryption_algorithm); + int encryptedSize = encrypt_data((unsigned char *)data, data_size, aesCbcEncryptionKey, encryption_type, + processed_data, column_encryption_algorithm); return encryptedSize; } @@ -203,7 +202,7 @@ DecryptDataRes EncryptionColumnHookExecutor::deprocess_data_impl(const unsigned } res = memset_s(*data, data_proceeed_size, 0, data_proceeed_size); securec_check_c(res, "\0", "\0"); - int plainTextSize = decrypt_data(data_processed, data_proceeed_size, aesCbcEncryptionKey, *data, + int plainTextSize = decrypt_data((unsigned char *)data_processed, data_proceeed_size, aesCbcEncryptionKey, *data, column_encryption_algorithm); if (plainTextSize <= 0) { printfPQExpBuffer(&conn->errorMessage, libpq_gettext("ERROR(CLIENT): failed to decrypt data.\n")); @@ -230,36 +229,29 @@ bool EncryptionColumnHookExecutor::deprocess_column_encryption_key( const char *key_store_str = NULL; const char *key_path_str = NULL; const char *key_algo_str = NULL; - CmkemErrCode ret = CMKEM_SUCCEED; - CmkemUStr cek_cipher = {(unsigned char *)encrypted_key_value, *encrypted_key_value_size}; - CmkemUStr *cek_plain = NULL; encryption_global_hook_executor->get_argument("key_store", &key_store_str, unused); encryption_global_hook_executor->get_argument("key_path", &key_path_str, unused); encryption_global_hook_executor->get_argument("algorithm", &key_algo_str, unused); - CmkIdentity cmk_identify = {key_store_str, key_path_str, key_algo_str, 0, PgClientLogic.client_cache_id}; - - ret = reg_all_cmk_entity_manager(); - if (ret != CMKEM_SUCCEED) { - printfPQExpBuffer(&conn->errorMessage, "ERROR(CLIENT): %s\n", get_cmkem_errmsg(ret)); - return false; - } - - ret = decrypt_cek_cipher(&cek_cipher, &cmk_identify, &cek_plain); - if (ret != CMKEM_SUCCEED) { - printfPQExpBuffer(&conn->errorMessage, "ERROR(CLIENT): %s\n", get_cmkem_errmsg(ret)); + KeyAdpt *adpt = CL_GET_KEY_ADPT(PgClientLogic); + KeyInfo info = {key_store_str, key_path_str, key_algo_str}; + KmUnStr cipher = {(unsigned char *)encrypted_key_value, *encrypted_key_value_size}; + KmUnStr plain = {0}; + plain = key_adpt_mk_decrypt(adpt, info, cipher); + if (key_adpt_catch_err(adpt)) { + printfPQExpBuffer(&conn->errorMessage, "ERROR(CLIENT): %s\n", key_adpt_get_err(adpt)); conn->client_logic->is_external_err = true; return false; } size_t i = 0; - for (; i < cek_plain->ustr_len && i < MAX_CEK_LENGTH - 1; i++) { - decrypted_key[i] = cek_plain->ustr_val[i]; + for (; i < plain.len && i < MAX_CEK_LENGTH - 1; i++) { + decrypted_key[i] = plain.val[i]; } - decrypted_key[i] = '\0'; *decrypted_key_size = i; - free_cmkem_ustr_with_erase(cek_plain); + + free(plain.val); return true; } @@ -339,14 +331,11 @@ bool EncryptionColumnHookExecutor::pre_create(PGClientLogic &column_encryption, const char *cmk_store_str = NULL; const char *cmk_path_str = NULL; const char *cmk_algo_str = NULL; - CmkemErrCode ret = CMKEM_SUCCEED; encryption_global_hook_executor->get_argument("key_store", &cmk_store_str, unused); encryption_global_hook_executor->get_argument("key_path", &cmk_path_str, unused); encryption_global_hook_executor->get_argument("algorithm", &cmk_algo_str, unused); - CmkIdentity cmk_identify = {cmk_store_str, cmk_path_str, cmk_algo_str, 0, column_encryption.client_cache_id}; - ColumnEncryptionAlgorithm column_encryption_algorithm(ColumnEncryptionAlgorithm::INVALID_ALGORITHM); char *expected_value = NULL; const char *expected_value_find = NULL; @@ -390,29 +379,24 @@ bool EncryptionColumnHookExecutor::pre_create(PGClientLogic &column_encryption, } /* encrypt CEK */ - CmkemUStr cek_plain = {(unsigned char *)expected_value, expected_value_size}; - CmkemUStr *cek_cipher = NULL; - - ret = reg_all_cmk_entity_manager(); - if (ret != CMKEM_SUCCEED) { - printfPQExpBuffer(&conn->errorMessage, "ERROR(CLIENT): %s\n", get_cmkem_errmsg(ret)); - free_after_check(!has_user_set_cek, (unsigned char *)expected_value, expected_value_size); - return false; - } - - ret = encrypt_cek_plain(&cek_plain, &cmk_identify, &cek_cipher); + /* encrypt CEK */ + KeyAdpt *adpt = CL_GET_KEY_ADPT(column_encryption); + KeyInfo info = {cmk_store_str, cmk_path_str, cmk_algo_str}; + KmUnStr plain = {(unsigned char *)expected_value, expected_value_size}; + KmUnStr cipher = {0}; + cipher = key_adpt_mk_encrypt(adpt, info, plain); free_after_check(!has_user_set_cek, (unsigned char *)expected_value, expected_value_size); - if (ret != CMKEM_SUCCEED) { - printfPQExpBuffer(&conn->errorMessage, "ERROR(CLIENT): %s\n", get_cmkem_errmsg(ret)); + if (key_adpt_catch_err(adpt)) { + printfPQExpBuffer(&conn->errorMessage, "ERROR(CLIENT): %s\n", key_adpt_get_err(adpt)); conn->client_logic->is_external_err = true; return false; } /* escape encrypted CEK for BYTEA insertion */ size_t encoded_encrypted_value_size; - char *encoded_encrypted_value = (char *)PQescapeByteaConn(conn, cek_cipher->ustr_val, - cek_cipher->ustr_len, &encoded_encrypted_value_size); - free_cmkem_ustr(cek_cipher); + char *encoded_encrypted_value = (char *)PQescapeByteaConn(conn, cipher.val, cipher.len, + &encoded_encrypted_value_size); + free(cipher.val); if (encoded_encrypted_value == NULL) { printfPQExpBuffer(&conn->errorMessage, libpq_gettext("ERROR(CLIENT): failed to encode encrypted values.\n")); return false; diff --git a/src/common/interfaces/libpq/client_logic_hooks/encryption_hooks/encryption_column_hook_executor.h b/src/common/interfaces/libpq/client_logic_hooks/encryption_hooks/encryption_column_hook_executor.h index 5adcd1745..33fe6501f 100644 --- a/src/common/interfaces/libpq/client_logic_hooks/encryption_hooks/encryption_column_hook_executor.h +++ b/src/common/interfaces/libpq/client_logic_hooks/encryption_hooks/encryption_column_hook_executor.h @@ -32,8 +32,8 @@ #include "column_hook_executor.h" #include "abstract_encryption_hook.h" #include "encryption_global_hook_executor.h" -#include "encrypt_decrypt.h" -#include "aead_aes_hamc_enc_key.h" +#include "keymgr/encrypt/security_encrypt_decrypt.h" +#include "keymgr/encrypt/security_aead_aes_hamc_enc_key.h" #include "postgres_ext.h" #include "client_logic_processor/values_processor.h" diff --git a/src/common/interfaces/libpq/client_logic_hooks/encryption_hooks/encryption_global_hook_executor.cpp b/src/common/interfaces/libpq/client_logic_hooks/encryption_hooks/encryption_global_hook_executor.cpp index 40dcc5277..f518048ef 100644 --- a/src/common/interfaces/libpq/client_logic_hooks/encryption_hooks/encryption_global_hook_executor.cpp +++ b/src/common/interfaces/libpq/client_logic_hooks/encryption_hooks/encryption_global_hook_executor.cpp @@ -35,9 +35,7 @@ #include "libpq-fe.h" #include "libpq-int.h" #include "cl_state.h" -#include "cmk_entity_manager_hooks/reg_cmkem_manager_main.h" -#include "cmk_entity_manager_hooks/reg_hook_frame.h" - +#include "keymgr/security_key_adpt.h" /* * while using SQL : * CREATE CLIENT MASTER KEY xxx WITH( @@ -59,30 +57,25 @@ bool EncryptionGlobalHookExecutor::pre_create(const StringArgs &args, const char *key_store_str = args.find("key_store"); const char *key_path_str = args.find("key_path"); const char *key_algo_str = args.find("algorithm"); - CmkIdentity cmk_identify = {key_store_str, key_path_str, key_algo_str, 0, m_clientLogic.client_cache_id}; - CmkemErrCode ret = CMKEM_SUCCEED; - /* - * You can register your own cmk entity management tools/components/services in reg_all_cmk_entity_manager() - * now, we have registered 3: - * (1) cmk entity management tool : gs_ktool. (not supported in openGauss) - * (2) cmk entity management component : localkms - * (3) cmk entity management service : HuaWei KMS (provided by HuaWei Cloud, not supported in openGauss) - * - * (*4) finally, we have a grammar check ：if cmk entity is not processed, we think it's out of control - * and it is a syntax error - */ - ret = reg_all_cmk_entity_manager(); - if (ret != CMKEM_SUCCEED) { - printfPQExpBuffer(&conn->errorMessage, "ERROR(CLIENT): %s\n", get_cmkem_errmsg(ret)); + KeyAdpt *adpt = CL_GET_KEY_ADPT(m_clientLogic); + KeyInfo info = {key_store_str, key_path_str, key_algo_str}; + char *keystate = key_adpt_mk_select(adpt, info); + if (key_adpt_catch_err(adpt)) { + libpq_free(keystate); + printfPQExpBuffer(&conn->errorMessage, "ERROR(CLIENT): %s\n", key_adpt_get_err(adpt)); + conn->client_logic->is_external_err = true; return false; } - ret = create_cmk_obj(&cmk_identify); - if (ret != CMKEM_SUCCEED) { - printfPQExpBuffer(&conn->errorMessage, "ERROR(CLIENT): %s\n", get_cmkem_errmsg(ret)); + if (keystate != NULL && strcmp(keystate, "active") != 0) { + printfPQExpBuffer(&conn->errorMessage, "ERROR(CLIENT): the key state of '%s' is not 'active' but '%s'\n", + key_path_str, keystate); + conn->client_logic->is_external_err = true; + libpq_free(keystate); return false; } + libpq_free(keystate); /* check same attributes are not used by existing global settings */ for (size_t i = 0; i < existing_global_hook_executors_size; ++i) { @@ -116,18 +109,13 @@ bool EncryptionGlobalHookExecutor::post_create(const StringArgs& args) const char *key_store_str = args.find("key_store"); const char *key_path_str = args.find("key_path"); const char *key_algo_str = args.find("algorithm"); - CmkIdentity cmk_identify = {key_store_str, key_path_str, key_algo_str, 0, m_clientLogic.client_cache_id}; - CmkemErrCode ret = CMKEM_SUCCEED; - - ret = reg_all_cmk_entity_manager(); - if (ret != CMKEM_SUCCEED) { - printfPQExpBuffer(&conn->errorMessage, "ERROR(CLIENT): %s\n", get_cmkem_errmsg(ret)); - return false; - } - ret = post_create_cmk_obj(&cmk_identify); - if (ret != CMKEM_SUCCEED) { - printfPQExpBuffer(&conn->errorMessage, "ERROR(CLIENT): %s\n", get_cmkem_errmsg(ret)); + /* encrypt CEK */ + KeyAdpt *adpt = CL_GET_KEY_ADPT(m_clientLogic); + KeyInfo info = {key_store_str, key_path_str, key_algo_str}; + key_adpt_mk_create(adpt, info); + if (key_adpt_catch_err(adpt)) { + printfPQExpBuffer(&conn->errorMessage, "ERROR(CLIENT): %s\n", key_adpt_get_err(adpt)); return false; } @@ -146,22 +134,16 @@ bool EncryptionGlobalHookExecutor::set_deletion_expected() size_t unused = 0; const char *key_store_str = NULL; const char *key_path_str = NULL; - CmkemErrCode ret = CMKEM_SUCCEED; get_argument("key_store", &key_store_str, unused); get_argument("key_path", &key_path_str, unused); - CmkIdentity cmk_identify = {key_store_str, key_path_str, NULL, 0, m_clientLogic.client_cache_id}; - - ret = reg_all_cmk_entity_manager(); - if (ret != CMKEM_SUCCEED) { - printfPQExpBuffer(&conn->errorMessage, "ERROR(CLIENT): %s\n", get_cmkem_errmsg(ret)); - return false; - } - - ret = drop_cmk_obj(&cmk_identify); - if (ret != CMKEM_SUCCEED) { - printfPQExpBuffer(&conn->errorMessage, "ERROR(CLIENT): %s\n", get_cmkem_errmsg(ret)); + /* encrypt CEK */ + KeyAdpt *adpt = CL_GET_KEY_ADPT(m_clientLogic); + KeyInfo info = {key_store_str, key_path_str, NULL}; + key_adpt_mk_delete(adpt, info); + if (key_adpt_catch_err(adpt)) { + printfPQExpBuffer(&conn->errorMessage, "ERROR(CLIENT): %s\n", key_adpt_get_err(adpt)); return false; } @@ -233,25 +215,20 @@ bool EncryptionGlobalHookExecutor::deprocess_column_setting(const unsigned char size_t processed_data_size, const char *key_store, const char *key_path, const char *key_algo, unsigned char **data, size_t *data_size) { - CmkIdentity cmk_identify = {key_store, key_path, key_algo, 0, 1}; - CmkemUStr cek_cipher = {(unsigned char *)processed_data, (size_t)processed_data_size}; - CmkemUStr *cek_plain = NULL; - CmkemErrCode ret = CMKEM_SUCCEED; - - ret = reg_all_cmk_entity_manager(); - if (ret != CMKEM_SUCCEED) { - /* this function is only called by gs_dump, so we can print errmsg on the terminal */ - printf("ERROR: %s", get_cmkem_errmsg(ret)); - return false; - } - - if (decrypt_cek_cipher(&cek_cipher, &cmk_identify, &cek_plain) != CMKEM_SUCCEED) { + KeyAdpt *adpt = key_adpt_new(); + KeyInfo info = {key_store, key_path, key_algo}; + KmUnStr cipher = {(unsigned char *)processed_data, (size_t)processed_data_size}; + KmUnStr plain = {0}; + plain = key_adpt_mk_decrypt(adpt, info, cipher); + if (key_adpt_catch_err(adpt)) { + printf("ERROR: %s\n", key_adpt_get_err(adpt)); + key_adpt_free(adpt); return false; } - *data = cek_plain->ustr_val; - *data_size = cek_plain->ustr_len; + *data = plain.val; + *data_size = plain.len; + key_adpt_free(adpt); - cmkem_free(cek_plain); return true; } diff --git a/src/common/interfaces/libpq/frontend_parser/gram.y b/src/common/interfaces/libpq/frontend_parser/gram.y index ebe4ad7a7..b366d2eb2 100755 --- a/src/common/interfaces/libpq/frontend_parser/gram.y +++ b/src/common/interfaces/libpq/frontend_parser/gram.y @@ -5324,7 +5324,7 @@ master_key_elem: // len is not filled on purpose ?? $$ = (Node*) n; } - | KEY_PATH '=' ColId + | KEY_PATH '=' ColId_or_Sconst { ClientLogicGlobalParam *n = makeNode (ClientLogicGlobalParam); n->key = ClientLogicGlobalProperty::CMK_KEY_PATH; diff --git a/src/common/interfaces/libpq/jdbc/client_logic_jni/client_logic_jni.h b/src/common/interfaces/libpq/jdbc/client_logic_jni/client_logic_jni.h index e10fe4117..5501bb950 100644 --- a/src/common/interfaces/libpq/jdbc/client_logic_jni/client_logic_jni.h +++ b/src/common/interfaces/libpq/jdbc/client_logic_jni/client_logic_jni.h @@ -27,7 +27,6 @@ #include "client_logic_common/statement_data.h" #include "driver_error.h" #include "jni_string_convertor.h" -#include "client_logic_hooks/cmk_entity_manager_hooks/register_huawei_kms.h" typedef struct pg_conn PGconn; /* forward declaration */ diff --git a/src/common/interfaces/libpq/jdbc/client_logic_jni/org_postgresql_jdbc_ClientLogicImpl.cpp b/src/common/interfaces/libpq/jdbc/client_logic_jni/org_postgresql_jdbc_ClientLogicImpl.cpp index 5a835a2a3..90dcfd266 100644 --- a/src/common/interfaces/libpq/jdbc/client_logic_jni/org_postgresql_jdbc_ClientLogicImpl.cpp +++ b/src/common/interfaces/libpq/jdbc/client_logic_jni/org_postgresql_jdbc_ClientLogicImpl.cpp @@ -209,7 +209,7 @@ JNIEXPORT jobjectArray JNICALL Java_org_postgresql_jdbc_ClientLogicImpl_setKmsIn if (client_logic_jni->set_kms_info(key.c_str, value.c_str)) { result.set_no_error_retrun(); } else { - status.set_error(INVALID_INPUT_PARAMETER, get_cmkem_errmsg(CMKEM_CHECK_INPUT_AUTH_ERR)); + status.set_error(INVALID_INPUT_PARAMETER, "set kms info error"); result.set_error_return(&status); } diff --git a/src/gausskernel/security/CMakeLists.txt b/src/gausskernel/security/CMakeLists.txt index 960ac152b..736d4e12f 100755 --- a/src/gausskernel/security/CMakeLists.txt +++ b/src/gausskernel/security/CMakeLists.txt @@ -9,10 +9,12 @@ set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/iprange ${CMAKE_CURRENT_SOURCE_DIR}/gs_ledger ${CMAKE_CURRENT_SOURCE_DIR}/tde_key_management + ${CMAKE_CURRENT_SOURCE_DIR}/keymgr ) add_subdirectory(gs_policy) add_subdirectory(iprange) add_subdirectory(gs_ledger) +add_subdirectory(keymgr) add_subdirectory(tde_key_management) diff --git a/src/gausskernel/security/Makefile b/src/gausskernel/security/Makefile index b86539bbd..646ba7d90 100644 --- a/src/gausskernel/security/Makefile +++ b/src/gausskernel/security/Makefile @@ -8,6 +8,6 @@ subdir = src/gausskernel/security top_builddir = ../../.. include $(top_builddir)/src/Makefile.global -SUBDIRS = iprange gs_policy gs_ledger tde_key_management +SUBDIRS = iprange gs_policy gs_ledger tde_key_management # keymgr include $(top_srcdir)/src/gausskernel/common.mk diff --git a/src/gausskernel/security/keymgr/CMakeLists.txt b/src/gausskernel/security/keymgr/CMakeLists.txt new file mode 100755 index 000000000..2a157452a --- /dev/null +++ b/src/gausskernel/security/keymgr/CMakeLists.txt @@ -0,0 +1,90 @@ +#This is the main CMAKE for build all components. +set(ENABLE_KT "OFF") + +if(NOT "${ENABLE_MULTIPLE_NODES}_${ENABLE_PRIVATEGAUSS}" STREQUAL "OFF_OFF") + set(ENABLE_KT "ON") +endif() + +if("${ENABLE_LITE_MODE}" STREQUAL "ON") + set(ENABLE_KT "OFF") +endif() + +# 1. build libkeymgr.so and libkeymgr.a +# source +AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR}/comm TGT_libkey_SRC) +AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR}/encrypt TGT_libkey_SRC) +AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR}/his TGT_libkey_SRC) +AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR}/hwc TGT_libkey_SRC) +AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR}/localkms TGT_libkey_SRC) +AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR}/api TGT_libkey_SRC) +if("${ENABLE_KT}" STREQUAL "ON") + AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR}/ktool TGT_libkey_SRC) +endif() +# include +set(TGT_libkey_INC + ${PROJECT_SRC_DIR}/incldue/keymgr + ${PROJECT_SRC_DIR}/incldue/keymgr/encrypt + ${PROJECT_TRUNK_DIR}/distribute/include + ${CJSON_INCLUDE_PATH} + ${CURL_INCLUDE_PATH} + ${LIBOPENSSL_INCLUDE_PATH}) +if("${ENABLE_KT}" STREQUAL "ON") + list(APPEND TGT_libkey_INC ${KMC_INCLUDE_PATH}) +endif() +# c flage +set(libkey_DEF_OPTIONS ${MACRO_OPTIONS}) +set(libkey_COMPILE_OPTIONS + ${OPTIMIZE_OPTIONS} + ${OS_OPTIONS} + ${PROTECT_OPTIONS} + ${WARNING_OPTIONS} + ${LIB_SECURE_OPTIONS} + ${CHECK_OPTIONS} + -DFRONTEND + -fstack-protector-strong) +if("${ENABLE_KT}" STREQUAL "ON") + list(APPEND libkey_COMPILE_OPTIONS -DENABLE_KT) +endif() +list(REMOVE_ITEM libkey_COMPILE_OPTIONS -fstack-protector) +# ld flags +set(libkey_LINK_OPTIONS -pthread -ftrapv -Wl,-z,relro,-z,now -Wl,-z,noexecstack) +# make +add_shared_libtarget(keymgr + TGT_libkey_SRC + TGT_libkey_INC + "${libkey_DEF_OPTIONS}" + "${libkey_COMPILE_OPTIONS}" + "${libkey_LINK_OPTIONS}") +add_static_libtarget(keymgr + TGT_libkey_SRC + TGT_libkey_INC + "${libkey_DEF_OPTIONS}" + "${libkey_COMPILE_OPTIONS}") +# ld libs +if("${ENABLE_KT}" STREQUAL "ON") + add_dependencies(keymgr gs_ktool) +endif() +set(libkey_LINKS -lcjson -lcurl -lsecurec -lssl -lcrypto -ldl -lrt) +if("${ENABLE_KT}" STREQUAL "ON") + list(APPEND libkey_LINKS -lgs_ktool -lkmc) +endif() +target_link_libraries(keymgr PUBLIC ${libkey_LINKS}) +# ld flags +set(libkey_LINKS_DIRS + ${LIBOPENSSL_LIB_PATH} + ${CJSON_LIB_PATH} + ${LIBCURL_LIB_PATH} + ${SECURE_LIB_PATH} + ${CMAKE_BINARY_DIR}/lib) +if("${ENABLE_KT}" STREQUAL "ON") + list(APPEND libkey_LINKS_DIRS ${KMC_LIB_PATH}) +endif() +target_link_directories(keymgr PUBLIC ${libkey_LINKS_DIRS}) +# install +install(TARGETS keymgr LIBRARY DESTINATION lib) +install(TARGETS keymgr ARCHIVE DESTINATION lib) + +set(keymgr_DEF_OPTIONS ${MACRO_OPTIONS}) +set(keymgr_COMPILE_OPTIONS ${OPTIMIZE_OPTIONS} ${OS_OPTIONS} ${PROTECT_OPTIONS} ${WARNING_OPTIONS} ${BIN_SECURE_OPTIONS} ${CHECK_OPTIONS}) +set(keymgr_LINK_OPTIONS ${BIN_LINK_OPTIONS}) +add_static_objtarget(gausskernel_security_keymgr TGT_libkey_SRC TGT_libkey_INC "${keymgr_DEF_OPTIONS}" "${keymgr_COMPILE_OPTIONS}" "${keymgr_LINK_OPTIONS}") \ No newline at end of file diff --git a/src/gausskernel/security/keymgr/Makefile b/src/gausskernel/security/keymgr/Makefile new file mode 100644 index 000000000..1ee26cc1f --- /dev/null +++ b/src/gausskernel/security/keymgr/Makefile @@ -0,0 +1,41 @@ +# +# Copyright (c) 2020 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# --------------------------------------------------------------------------------------- +# +# Makefile +# Makefile for keymgr +# +# IDENTIFICATION +# src/gausskernel/security/keymgr/Makefile +# +# --------------------------------------------------------------------------------------- + +subdir=src/gausskernel/security/keymgr +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global + +#/src/include/event/suse + +ifneq "$(MAKECMDGOALS)" "clean" + ifneq "$(MAKECMDGOALS)" "distclean" + ifneq "$(shell which g++ |grep hutaf_llt |wc -l)" "1" + -include $(DEPEND) + endif + endif +endif + +FIELS = $(shell find $(top_builddir)/$(subdir) -name '*.cpp' | sort) +OBJS = $(FIELS:.cpp=.o) + +include $(top_srcdir)/src/gausskernel/common.mk diff --git a/src/gausskernel/security/keymgr/api/security_key_adpt.cpp b/src/gausskernel/security/keymgr/api/security_key_adpt.cpp new file mode 100644 index 000000000..1a1e66d1e --- /dev/null +++ b/src/gausskernel/security/keymgr/api/security_key_adpt.cpp @@ -0,0 +1,418 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * security_key_adpt.cpp + * you can register your own key management tool/component/service here. + * what's more, you can choose which to use according to the version and usage scenarios. + * + * IDENTIFICATION + * src/gausskernel/security/keymgr/src/keymgr/security_key_adpt.cpp + * + * ------------------------------------------------------------------------- + */ +#include "keymgr/security_key_adpt.h" +#include +#include +#include "keymgr/comm/security_version.h" +#include "keymgr/hwc/security_hwc.h" +#include "keymgr/comm/security_utils.h" +#include "keymgr/his/security_his.h" +#include "keymgr/localkms/security_localkms.h" +#include "keymgr/ktool/security_gs_ktool.h" + +KeyMethod *g_key_method[MAX_KEY_MGR_NUM] = { + &his_kms, /* HisAccount, HisSecret, HisAppId, HisEnterprise, HisIamUrl, HisKmsUrl, */ + +#ifndef FRONTEND + &hcs_kms, +#endif + +#ifdef ENABLE_HUAWEI_KMS + &huawei_kms, +#endif + +#ifdef ENABLE_LOCALKMS +#ifdef FRONTEND + &localkms, +#endif +#endif + +#ifdef ENABLE_GS_KTOOL +#ifdef FRONTEND + &gs_ktool, +#endif +#endif + + NULL, /* must end with 'NULL' */ +}; + +KeyAdpt *key_adpt_new() +{ +#define KEY_ADPT_ERR_BUF_SZ 4096 + KmErr *err; + KeyAdpt *adpt; + + err = km_err_new(KEY_ADPT_ERR_BUF_SZ); + if (err == NULL) { + return NULL; + } + + adpt = (KeyAdpt *)km_cxt_alloc_zero(err->cxt, sizeof(KeyAdpt)); + if (adpt == NULL) { + km_err_free(err); + return NULL; + } + adpt->err = err; + + for (int i = 0; g_key_method[i] != NULL; i++) { + adpt->methods[i] = g_key_method[i]; + } + + return adpt; +} + +void key_adpt_free(KeyAdpt *kadpt) +{ + if (kadpt == NULL) { + return; + } + + for (int i = 0; kadpt->methods[i] != NULL; i++) { + if (kadpt->methods[i]->kmgr_free != NULL && kadpt->mgrs[i] != NULL) { + kadpt->methods[i]->kmgr_free(kadpt->mgrs[i]); + kadpt->mgrs[i] = NULL; + } + } + km_err_free(kadpt->err); + km_free(kadpt); +} + +static int key_adpt_get_index(KeyAdpt *kadpt, const char *type) +{ + if (type == NULL) { + km_err_msg(kadpt->err, "fail to get key manager type."); + return -1; + } + + for (int i = 0; kadpt->methods[i] != NULL; i++) { + if (strcasecmp(kadpt->methods[i]->type, type) != 0) { + continue; + } +#ifndef FRONTEND + MemoryContext old = MemoryContextSwitchTo(kadpt->err->cxt); +#endif + if (kadpt->mgrs[i] == NULL) { + kadpt->mgrs[i] = kadpt->methods[i]->kmgr_new(kadpt->err); + } +#ifndef FRONTEND + MemoryContextSwitchTo(old); +#endif + if (kadpt->mgrs[i] == NULL) { + if (!km_err_catch(kadpt->err)) { + km_err_msg(kadpt->err, "failed to init key manager of '%s'.", type); + } + return -1; + } + return i; + } + + km_err_msg(kadpt->err, "unknown key manager type: '%s'.", type); + return -1; +} + +void key_adpt_set_arg(KeyAdpt *kadpt, const char *type, const char *key, const char *value) +{ + int idx; + KeyMgr *kmgr; + KeyMethod *kmethod; + + if (kadpt == NULL) { + return; + } + + idx = key_adpt_get_index(kadpt, type); + if (idx < 0) { + return; + } + + kmethod = kadpt->methods[idx]; + kmgr = kadpt->mgrs[idx]; + +#ifndef FRONTEND + MemoryContext old = MemoryContextSwitchTo(kadpt->err->cxt); +#endif + if (kmethod->kmgr_set_arg != NULL) { + kmethod->kmgr_set_arg(kmgr, key, value); + } +#ifndef FRONTEND + MemoryContextSwitchTo(old); +#endif +} + +void key_adpt_set_args(KeyAdpt *kadpt, const char *type, const char *args) +{ + int idx; + KeyMethod *kmethod; + KvScan *scan; + int kvcnt; + + if (kadpt == NULL) { + return; + } + + km_err_reset(kadpt->err); + idx = key_adpt_get_index(kadpt, type); + if (idx < 0) { + return; + } + + kmethod = kadpt->methods[idx]; + scan = kv_scan_init(args); + for (;;) { + kvcnt = kv_scan_exec(scan); + if (kvcnt >= 0) { + if (kmethod->kmgr_set_arg != NULL) { + kmethod->kmgr_set_arg(kadpt->mgrs[idx], km_str_strip(scan->key), km_str_strip(scan->value)); + } + } else { + km_err_msg(kadpt->err, "failed to parse key info of '%s', error at position %d.", type, scan->curpos); + } + + if (kvcnt <= 0) { + break; + } + } + kv_scan_drop(scan); +} + +void key_adpt_mk_create(KeyAdpt *kadpt, KeyInfo info) +{ + int idx; + KeyMgr *kmgr; + KeyMethod *kmethod; + + if (kadpt == NULL) { + return; + } + + km_err_reset(kadpt->err); + + idx = key_adpt_get_index(kadpt, info.type); + if (idx < 0) { + return; + } + + kmethod = kadpt->methods[idx]; + kmgr = kadpt->mgrs[idx]; + +#ifndef FRONTEND + MemoryContext old = MemoryContextSwitchTo(kadpt->err->cxt); +#endif + if (kmethod->mk_create != NULL) { + kmethod->mk_create(kmgr, info); + } +#ifndef FRONTEND + MemoryContextSwitchTo(old); +#endif +} + +void key_adpt_mk_delete(KeyAdpt *kadpt, KeyInfo info) +{ + int idx; + KeyMgr *kmgr; + KeyMethod *kmethod; + + if (kadpt == NULL) { + return; + } + + km_err_reset(kadpt->err); + + idx = key_adpt_get_index(kadpt, info.type); + if (idx < 0) { + return; + } + + kmethod = kadpt->methods[idx]; + kmgr = kadpt->mgrs[idx]; + +#ifndef FRONTEND + MemoryContext old = MemoryContextSwitchTo(kadpt->err->cxt); +#endif + if (kmethod->mk_delete != NULL) { + kmethod->mk_delete(kmgr, info); + } +#ifndef FRONTEND + MemoryContextSwitchTo(old); +#endif +} + +char *key_adpt_mk_select(KeyAdpt *kadpt, KeyInfo info) +{ + int idx; + KeyMgr *kmgr; + KeyMethod *kmethod; + char *keystate = NULL; + + if (kadpt == NULL) { + return NULL; + } + + km_err_reset(kadpt->err); + + idx = key_adpt_get_index(kadpt, info.type); + if (idx < 0) { + return NULL; + } + + kmethod = kadpt->methods[idx]; + kmgr = kadpt->mgrs[idx]; + +#ifndef FRONTEND + MemoryContext old = MemoryContextSwitchTo(kadpt->err->cxt); +#endif + if (kmethod->mk_select != NULL) { + keystate = kmethod->mk_select(kmgr, info); + } +#ifndef FRONTEND + MemoryContextSwitchTo(old); +#endif + + return keystate; +} + +KmUnStr key_adpt_mk_encrypt(KeyAdpt *kadpt, KeyInfo info, KmUnStr plain) +{ + int idx; + KeyMgr *kmgr; + KeyMethod *kmethod; + KmUnStr cipher = {0}; + + if (kadpt == NULL) { + return cipher; + } + + km_err_reset(kadpt->err); + + idx = key_adpt_get_index(kadpt, info.type); + if (idx < 0) { + return cipher; + } + + kmethod = kadpt->methods[idx]; + kmgr = kadpt->mgrs[idx]; + +#ifndef FRONTEND + MemoryContext old = MemoryContextSwitchTo(kadpt->err->cxt); +#endif + if (kmethod->mk_enrypt != NULL) { + cipher = kmethod->mk_enrypt(kmgr, info, plain); + } +#ifndef FRONTEND + MemoryContextSwitchTo(old); +#endif + + return cipher; +} + +KmUnStr key_adpt_mk_decrypt(KeyAdpt *kadpt, KeyInfo info, KmUnStr cipher) +{ + int idx; + KeyMgr *kmgr; + KeyMethod *kmethod; + KmUnStr plain = {0}; + + if (kadpt == NULL) { + return plain; + } + + km_err_reset(kadpt->err); + + idx = key_adpt_get_index(kadpt, info.type); + if (idx < 0) { + return plain; + } + + kmethod = kadpt->methods[idx]; + kmgr = kadpt->mgrs[idx]; + +#ifndef FRONTEND + MemoryContext old = MemoryContextSwitchTo(kadpt->err->cxt); +#endif + if (kmethod->mk_decrypt != NULL) { + plain = kmethod->mk_decrypt(kmgr, info, cipher); + } +#ifndef FRONTEND + MemoryContextSwitchTo(old); +#endif + + return plain; +} + +KmUnStr key_adpt_dk_create(KeyAdpt *kadpt, KeyInfo info, KmUnStr *cipher) +{ + int idx; + KeyMgr *kmgr; + KeyMethod *kmethod; + KmUnStr plain = {0}; + + if (kadpt == NULL) { + return plain; + } + + km_err_reset(kadpt->err); + + idx = key_adpt_get_index(kadpt, info.type); + if (idx < 0) { + return plain; + } + + kmethod = kadpt->methods[idx]; + kmgr = kadpt->mgrs[idx]; + +#ifndef FRONTEND + MemoryContext old = MemoryContextSwitchTo(kadpt->err->cxt); +#endif + if (kmethod->dk_create != NULL) { + plain = kmethod->dk_create(kmgr, info, cipher); + } +#ifndef FRONTEND + MemoryContextSwitchTo(old); +#endif + + return plain; +} + +bool key_adpt_catch_err(KeyAdpt *kadpt) +{ + if (kadpt == NULL) { + return true; + } + + return km_err_catch(kadpt->err); +} + +const char *key_adpt_get_err(KeyAdpt *kadpt) +{ + if (kadpt == NULL) { + return "key adaptor is null."; + } + + if (strlen(kadpt->err->buf) != 0) { + return kadpt->err->buf; + } + + return "key adaptor: unknown error."; +} diff --git a/src/gausskernel/security/keymgr/comm/security_aksk.cpp b/src/gausskernel/security/keymgr/comm/security_aksk.cpp new file mode 100644 index 000000000..fab8357d8 --- /dev/null +++ b/src/gausskernel/security/keymgr/comm/security_aksk.cpp @@ -0,0 +1,238 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * security_aksk.cpp + * + * IDENTIFICATION + * src/gausskernel/security/keymgr/comm/security_aksk.cpp + * + * ------------------------------------------------------------------------- + */ + +#include "keymgr/comm/security_aksk.h" +#include +#include +#include +#include +#include +#include "keymgr/comm/security_error.h" +#include "keymgr/comm/security_encode.h" + +void km_aksk_add_reqhdr(AkSkPara *para, const char *hdr) +{ + if (para->hdrcnt >= KM_MAX_HTTP_HDR_NUM - 1) { + return; + } + para->reqhdrs[para->hdrcnt++] = hdr; +} + +/* + * input: http headers, e.g + * Header1: value1 + * Header2: some white-space space at the header + * Header3: value3 + * output: snapshort of http headers, e.g + * header1;header2;header3 + */ +void km_aksk_format_reqhdr(AkSkPara *para, char *buf, size_t bufsz) +{ + size_t i; + size_t j; + size_t bufuse = 0; + const char *hdr; + + for (i = 0; i < para->hdrcnt; i++) { + hdr = para->reqhdrs[i]; + for (; hdr[0] == ' '; hdr++) {}; + for (j = 0; j < strlen(hdr); j++) { + if (bufuse >= bufsz) { + break; + } else if (hdr[j] == ' ' || hdr[j] == ':') { + buf[bufuse++] = ';'; + break; + } else { + buf[bufuse++] = tolower(hdr[j]); + } + } + } + + if (bufuse > 0) { + buf[bufuse - 1] = '\0'; + } +} + +/* please make sure the format of header is 'key: value', do not add any white-space before ':' */ +int km_hdr_cmp_hook(const void *header1, const void *header2) +{ + const char *hdr1; + const char *hdr2; + char *key1 = NULL; + char *key2 = NULL; + size_t len1; + size_t len2; + int ret = -1; + + if (header1 == NULL || header2 == NULL) { + return -1; + } + + hdr1 = *(char **)header1; + hdr2 = *(char **)header2; + + for (len1 = 0; len1 < strlen(hdr1); len1++) { + if (hdr1[len1] == ':') { + key1 = km_strndup(hdr1, len1); + break; + } + } + for (len2 = 0; len2 < strlen(hdr2); len2++) { + if (hdr2[len2] == ':') { + key2 = km_strndup(hdr2, len2); + break; + } + } + if (key1 != NULL && key2 != NULL) { + ret = strcasecmp(key1, key2); + } + km_safe_free(key1); + km_safe_free(key2); + + return ret; +} + +AkSkSign km_aksk_sign(AkSkPara *para) +{ +#define KM_TIME_HDR_SZ 60 +#define KM_YEAR_UTC_OFF 1900 +#define KM_AUTH_HDR_SZ 1024 +#define KM_ALL_REQ_HDR 256 +#define KM_TOTAL_BUF_SZ 2048 +#define KM_SIGN_LEN 128 + + AkSkSign sign = {0}; + time_t current; + struct tm cur; + errno_t rc; + + /* add http header: host */ + size_t hostsz = strlen("Host: __") + strlen(para->host); + sign.host = (char *)km_alloc(hostsz); + rc = sprintf_s(sign.host, hostsz, "Host: %s", para->host); + km_securec_check_ss(rc, "", ""); + km_aksk_add_reqhdr(para, sign.host); + + /* add http header: date */ + sign.date = (char *)km_alloc(KM_TIME_HDR_SZ); + current = time(NULL); + if (gmtime_r(¤t, &cur) == NULL) { + return sign; + } + char fmtdate[KM_TIME_HDR_SZ] = {0}; + rc = sprintf_s(fmtdate, KM_TIME_HDR_SZ, "%04d%02d%02dT%02d%02d%02dZ", + cur.tm_year + KM_YEAR_UTC_OFF, cur.tm_mon + 1, cur.tm_mday, cur.tm_hour, cur.tm_min, cur.tm_sec); + km_securec_check_ss(rc, "", ""); + rc = sprintf_s(sign.date, KM_TIME_HDR_SZ, "X-Sdk-Date: %s", fmtdate); + km_securec_check_ss(rc, "", ""); + km_aksk_add_reqhdr(para, sign.date); + + qsort(para->reqhdrs, para->hdrcnt, sizeof(const char *), km_hdr_cmp_hook); + + /* + * httpbuf = + * method\n + * uri\n + * query\n + * header1\n + * header2\n + * \n + * header1:header2\n + * hex(sha(body)) + */ + char *httpbuf = (char *)km_alloc(KM_TOTAL_BUF_SZ); + char *buf = httpbuf; + size_t bufuse = 0; + size_t i; + size_t j; + char hdrshot[KM_ALL_REQ_HDR]; + + /* method */ + for (i = 0; i < strlen(para->method); i++) { + buf[bufuse++] = para->method[i]; + } + buf[bufuse++] = '\n'; + /* uri */ + bufuse += km_url_encode(para->uri, buf + bufuse, KM_TOTAL_BUF_SZ - bufuse); + buf[bufuse++] = '\n'; + /* query, but we have no query, so skip it */ + buf[bufuse++] = '\n'; + /* all reqheaders */ + for (i = 0; i < para->hdrcnt; i++) { + bool iskey = true; + const char *hdr = para->reqhdrs[i]; + for (j = 0; j < strlen(hdr); j++) { + if (hdr[j] == ' ') { + continue; + } else if (hdr[j] == ':') { + iskey = false; + buf[bufuse++] = ':'; + } else { + buf[bufuse++] = iskey ? tolower(hdr[j]) : hdr[j]; + } + } + buf[bufuse++] = '\n'; + } + buf[bufuse++] = '\n'; + /* snapshot of reqheaders */ + km_aksk_format_reqhdr(para, hdrshot, KM_ALL_REQ_HDR); + for (i = 0; i < strlen(hdrshot); i++) { + buf[bufuse++] = hdrshot[i]; + } + buf[bufuse++] = '\n'; + /* http body -> sha -> hex */ + bufuse += km_hex_sha(para->reqbody, strlen(para->reqbody), buf + bufuse, KM_TOTAL_BUF_SZ - bufuse); + buf[bufuse] = '\0'; + + /* + * signin = + * SDK-HMAC-SHA256\n + * fmtdate\n + * hex(sha(httpbuf)) + * signout = HMAC(signin) + * signouthex = hex(signout) + */ + char bufsha[KM_SIGN_LEN]; + char signin[KM_SIGN_LEN]; + unsigned char signout[KM_SIGN_LEN]; + unsigned int signsz; + km_hex_sha(buf, bufuse, bufsha, KM_SIGN_LEN); + km_safe_free(httpbuf); + rc = sprintf_s(signin, KM_SIGN_LEN, "SDK-HMAC-SHA256\n%s\n%s", fmtdate, bufsha); + km_securec_check_ss(rc, "", ""); + HMAC(EVP_sha256(), para->sk, strlen(para->sk), (unsigned char *)signin, strlen(signin), signout, &signsz); + KmStr signouthex = km_hex_encode({signout, signsz}); + if (signouthex.val == NULL) { + return sign; + } + + /* add http header: authorization */ + sign.signature = (char *)km_alloc(KM_AUTH_HDR_SZ); + rc = sprintf_s(sign.signature, KM_AUTH_HDR_SZ, + "Authorization: SDK-HMAC-SHA256 Access=%s, SignedHeaders=%s, Signature=%s", para->ak, hdrshot, signouthex.val); + km_securec_check_ss(rc, "", ""); + km_safe_free(signouthex.val); + km_aksk_add_reqhdr(para, sign.signature); + + return sign; +} \ No newline at end of file diff --git a/src/gausskernel/security/keymgr/comm/security_encode.cpp b/src/gausskernel/security/keymgr/comm/security_encode.cpp new file mode 100644 index 000000000..c397e460c --- /dev/null +++ b/src/gausskernel/security/keymgr/comm/security_encode.cpp @@ -0,0 +1,227 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * security_encode.cpp + * + * IDENTIFICATION + * src/gausskernel/security/keymgr/src/comm/security_encode.cpp + * + * ------------------------------------------------------------------------- + */ + +#include "keymgr/comm/security_encode.h" +#include +#include +#include +#include +#include +#include +#include +#include "keymgr/comm/security_error.h" +#include "keymgr/comm/security_utils.h" + +#define HEX_CODE_LEN 2 +#define HEX_SHIFT_LEN 4 + +static int hex_atoi(char hex) +{ + if (hex >= '0' && hex <= '9') { + return hex - '0'; + } else if (hex >= 'a' && hex <= 'f') { + return (hex - 'a') + 0xa; + } else if (hex >= 'A' && hex <= 'F') { + return (hex - 'A') + 0xa; + } + + return 0; +} + +KmStr km_hex_encode(KmUnStr str) +{ + size_t i; + const char *hextbl = "0123456789abcdef"; + KmStr hex = {0}; + size_t hexlen = 0; + + hex.val = (char *)km_alloc(str.len * HEX_CODE_LEN + 1); + if (hex.val == NULL) { + return hex; + } + + for (i = 0; i < str.len; i++) { + hex.val[hexlen++] = hextbl[(str.val[i] & 0xf0) >> HEX_SHIFT_LEN]; + hex.val[hexlen++] = hextbl[str.val[i] & 0x0f]; + } + hex.val[hexlen] = '\0'; + hex.len = hexlen; + + return hex; +} + +KmUnStr km_hex_decode(KmStr hex) +{ + KmUnStr str = {0}; + size_t pos = 0; + /* for bit operation, we need to use 'unsigned int' */ + unsigned int low; + unsigned int high; + + str.val = (unsigned char *)km_alloc((size_t)hex.len / HEX_CODE_LEN + 1); + if (str.val == NULL) { + return str; + } + + for (; pos < hex.len; pos += HEX_CODE_LEN) { + high = (unsigned int)hex_atoi(hex.val[pos]); + low = (unsigned int)hex_atoi(hex.val[pos + 1]); + str.val[(int) pos / HEX_CODE_LEN] = ((high << HEX_SHIFT_LEN) & 0xff) ^ (low & 0xff); + } + + str.val[(int) pos / HEX_CODE_LEN] = '\0'; + str.len = pos / HEX_CODE_LEN; + + return str; +} + +/* + * - in : 'str' + * - out: 'str' + 'sha256' + * - do : claculate sha256 of string, and append the sha256 to string + */ +#define SHA256_LEN 32 +static KmUnStr sha_join(KmUnStr str) +{ + KmUnStr sha = {0}; + size_t i; + + sha.val = (unsigned char *)km_alloc(str.len + SHA256_LEN + 1); + if (sha.val == NULL) { + return sha; + } + + for (i = 0; i < str.len; i++) { + sha.val[i] = str.val[i]; + } + + (void)SHA256(str.val, str.len, sha.val + str.len); + sha.len = str.len + SHA256_LEN; + + return sha; +} + +static KmUnStr sha_strip(KmUnStr sha) +{ + unsigned char check[SHA256_LEN + 1]; + KmUnStr str = {0}; + size_t i; + + if (sha.len <= SHA256_LEN) { + return str; + } + + str.len = sha.len - SHA256_LEN; + + str.val = (unsigned char *)km_alloc(str.len + 1); + if (str.val == NULL) { + return str; + } + + (void)SHA256(sha.val, str.len, check); + for (i = 0; i < SHA256_LEN; i++) { + if (sha.val[i + str.len] != check[i]) { + (void)printf("check sha256 err\n"); + return str; + } + } + + for (i = 0; i < str.len; i++) { + str.val[i] = sha.val[i]; + } + + return str; +} + +KmStr km_sha_encode(KmUnStr str) +{ + KmUnStr sha = {0}; + KmStr shahex = {0}; + + sha = sha_join(str); + if (sha.val == NULL) { + return shahex; + } + + shahex = km_hex_encode(sha); + km_safe_free(sha.val); + + return shahex; +} + +KmUnStr km_sha_decode(KmStr shahex) +{ + KmUnStr sha = {0}; + KmUnStr str = {0}; + + sha = km_hex_decode(shahex); + if (sha.val == NULL) { + return str; + } + + str = sha_strip(sha); + km_safe_free(sha.val); + + return str; +} + +/* replace the special character with %* */ +size_t km_url_encode(const char *in, char *out, size_t outsz) +{ + const char *legal = ".~_-/"; + const char *hextbl = "0123456789ABCDEF"; + size_t outuse = 0; + + for (size_t i = 0; i < strlen(in) && outuse < outsz; i++) { + if (isalnum((unsigned char)in[i]) || strchr(legal, in[i]) != NULL) { + out[outuse++] = in[i]; + } else { + out[outuse++] = '%'; + out[outuse++] = hextbl[(unsigned char)in[i] >> HEX_SHIFT_LEN]; + out[outuse++] = hextbl[(unsigned char)in[i] & 0xf]; + } + } + + if (out[outuse - 1] != '/') { + out[outuse++] = '/'; + } + + return outuse; +} + +size_t km_hex_sha(const char *data, size_t datalen, char *hexbuf, size_t bufsz) +{ +#define KM_SHA256_LEN 32 + unsigned char sha[KM_SHA256_LEN]; + const char hextbl[] = "0123456789abcdef"; + size_t bufuse = 0; + + (void)SHA256((const unsigned char *)data, datalen, sha); + + for (size_t i = 0; i < KM_SHA256_LEN && bufuse < bufsz; i++) { + hexbuf[bufuse++] = hextbl[(sha[i] & 0xf0) >> HEX_SHIFT_LEN]; + hexbuf[bufuse++] = hextbl[sha[i] & 0x0f]; + } + hexbuf[bufuse] = '\0'; + return bufuse; +} \ No newline at end of file diff --git a/src/gausskernel/security/keymgr/comm/security_error.cpp b/src/gausskernel/security/keymgr/comm/security_error.cpp new file mode 100644 index 000000000..212637fbd --- /dev/null +++ b/src/gausskernel/security/keymgr/comm/security_error.cpp @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * security_error.cpp + * + * IDENTIFICATION + * src/gausskernel/security/keymgr/src/comm/security_error.cpp + * + * ------------------------------------------------------------------------- + */ + +#include "keymgr/comm/security_error.h" +#ifndef FRONTEND +#include "knl/knl_instance.h" +#include "utils/aset.h" +#endif +#include +KmErr *km_err_new(size_t sz) +{ + KmErr *err; + + if (sz == 0) { + return NULL; + } + +#ifndef FRONTEND + MemoryContext cxt; + MemoryContext old; + + cxt = AllocSetContextCreate( + g_instance.instance_context, + "key manager module", + ALLOCSET_DEFAULT_MINSIZE, + ALLOCSET_DEFAULT_MINSIZE, + ALLOCSET_DEFAULT_MINSIZE, + SHARED_CONTEXT); + + old = MemoryContextSwitchTo(cxt); +#endif + + err = (KmErr *)km_alloc(sizeof(KmErr)); + if (err == NULL) { + return NULL; + } + + err->buf = (char *)km_alloc(sz); + if (err->buf == NULL) { + km_free(err); + return NULL; + } + +#ifndef FRONTEND + MemoryContextSwitchTo(old); + err->cxt = cxt; +#endif + + err->sz = sz; + err->code = KM_ERR_INIT; + + km_err_reset(err); + + return err; +} + +void km_err_free(KmErr *err) +{ + if (err == NULL) { + return; + } + + km_free(err->buf); + km_free(err); +} + +#ifndef FRONTEND +void *km_cxt_alloc_zero(MemoryContext cxt, size_t size) +{ + void *buf; + MemoryContext old; + + old = MemoryContextSwitchTo(cxt); + buf = km_alloc_zero(size); + MemoryContextSwitchTo(old); + + return buf; +} +#endif \ No newline at end of file diff --git a/src/gausskernel/security/keymgr/comm/security_http.cpp b/src/gausskernel/security/keymgr/comm/security_http.cpp new file mode 100644 index 000000000..283962919 --- /dev/null +++ b/src/gausskernel/security/keymgr/comm/security_http.cpp @@ -0,0 +1,377 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * security_http.cpp + * + * IDENTIFICATION + * src/gausskernel/security/keymgr/src/comm/security_http.cpp + * + * ------------------------------------------------------------------------- + */ +#include "keymgr/comm/security_http.h" +#include +#include +#include "keymgr/comm/security_utils.h" +#include "keymgr/comm/security_httpscan.h" +#include "keymgr/comm/security_aksk.h" + +#define CHK_CURL(http, ret) \ + if ((ret) != CURLE_OK) { \ + km_err_msg((http)->err, "curl error, err code: %d, err message: %s", (ret), curl_easy_strerror((ret))); \ + } + +/* + * init curl once. see what the curl say: + * - "curl_global_init() should be invoked exactly once for each application" + * - "This function is not thread-safe!" + */ +static void httpmgr_init_curl(HttpMgr *http) +{ + static pthread_mutex_t initlock = PTHREAD_MUTEX_INITIALIZER; + static bool initcurl = false; + + (void)pthread_mutex_lock(&initlock); + if (!initcurl) { + CURLcode ret = curl_global_init(CURL_GLOBAL_ALL); + CHK_CURL(http, ret); + initcurl = true; + } + (void)pthread_mutex_unlock(&initlock); +} + +/* + * implemente based on libcurl, support : requset line, requeset header, request body + */ +HttpMgr *httpmgr_new(KmErr *errbuf) +{ +#define HTTP_TIME_OUT 15 + + HttpMgr *http = (HttpMgr *)km_alloc_zero(sizeof(HttpMgr)); + if (http == NULL) { + return NULL; + } + + http->err = errbuf; + + httpmgr_init_curl(http); + + http->client = curl_easy_init(); + if (http->client == NULL) { + km_free(http); + return NULL; + } + + http->resscan = resscan_new(http->err); + if (http->resscan == NULL) { + curl_easy_cleanup(http->client); + km_free(http); + return NULL; + } + http->timeout = HTTP_TIME_OUT; + http->sslverify = 0; + + (void)curl_easy_setopt(http->client, CURLOPT_TIMEOUT, http->timeout); + + return http; +} + +void httpmgr_free(HttpMgr *http) +{ + if (http == NULL) { + return; + } + + curl_easy_cleanup(http->client); + + km_safe_free(http->url); + + if (http->reqhdrs != NULL) { + curl_slist_free_all(http->reqhdrs); + } + + resscan_free(http->resscan); + http->resscan = NULL; + + km_safe_free(http->reshdrs); + km_safe_free(http->reqlen); + + km_safe_free(http->ak); + km_safe_free(http->sk); + + km_free(http); +} + +void httpmgr_set_req_line(HttpMgr *http, const char *url, CURLoption method, const char *cacert) +{ + bool setca; + + if (km_err_catch(http->err)) { + return; + } + + /* reset error buffer */ + if (url == NULL) { + km_err_msg(http->err, "set url to null"); + return; + } + km_safe_free(http->url); + + http->url = km_strdup(url); + if (http->url == NULL) { + km_err_msg(http->err, "failed to copy and store url"); + return; + } + + http->method = method; + + CURLcode ret = curl_easy_setopt(http->client, CURLOPT_HTTPPOST, http->method); + CHK_CURL(http, ret); + ret = curl_easy_setopt(http->client, CURLOPT_URL, http->url); + CHK_CURL(http, ret); + + setca = (cacert != NULL) ? true : false; + (void)curl_easy_setopt(http->client, CURLOPT_SSL_VERIFYPEER, (long)setca); + (void)curl_easy_setopt(http->client, CURLOPT_SSL_VERIFYHOST, (long)setca); + + if (setca) { + (void)curl_easy_setopt(http->client, CURLOPT_CAINFO, cacert); + } + +#ifdef ENABLE_KM_DEBUG + resscan_output(http->resscan, "@request\n"); + resscan_output(http->resscan, url); + resscan_output(http->resscan, "\n"); +#endif +} + +/* + * you can call this function more than once. + * we will free all headers at each call of httpmgr_receive() + */ +void httpmgr_set_req_header(HttpMgr *http, const char *header) +{ + if (km_err_catch(http->err)) { + return; + } + + if (header == NULL) { + km_err_msg(http->err, "set http request header to null"); + return; + } + http->reqhdrs = curl_slist_append(http->reqhdrs, header); + if (http->reqhdrs == NULL) { + km_err_msg(http->err, "failed to set http request header"); + } + +#ifdef ENABLE_KM_DEBUG + resscan_output(http->resscan, header); + resscan_output(http->resscan, "\n"); +#endif +} + +char *httpmgr_get_req_body_len(HttpMgr *http, char *reqbody) +{ +#define REQ_LEN_BUF_SZ 64 + errno_t rc = 0; + + if (reqbody == NULL) { + return NULL; + } + + if (http->reqlen == NULL) { + http->reqlen = (char *)km_alloc(REQ_LEN_BUF_SZ); + } + + if (http->reqlen == NULL) { + return NULL; + } + + rc = sprintf_s(http->reqlen, REQ_LEN_BUF_SZ, "Content-Length: %lu", strlen(reqbody)); + km_securec_check_ss(rc, "", ""); + + return http->reqlen; +} + +/* called only once */ +void httpmgr_set_req_body(HttpMgr *http, char *reqbody) +{ + if (km_err_catch(http->err)) { + return; + } + if (reqbody == NULL) { + km_err_msg(http->err, "set http requet body to null"); + } + http->reqbody = reqbody; + CURLcode ret = curl_easy_setopt(http->client, CURLOPT_POSTFIELDS, reqbody); + CHK_CURL(http, ret); + +#ifdef ENABLE_KM_DEBUG + resscan_output(http->resscan, reqbody); + resscan_output(http->resscan, "\n"); +#endif +} + +/* + * remain[i] : if copy field in http response + * if 1: copy, and store in http->resscan + * if 0: do not copy, which means we will lose this field + * + * remain[0] : if copy response line + * remain[1] : if copy response header + * remain[2] : if copy response body + */ +void httpmgr_set_response(HttpMgr *http, char remain[3], const char *hdrkey) +{ + if (km_err_catch(http->err)) { + return; + } + + resscan_set_receive(http->resscan, remain, hdrkey); +} + +static size_t http_res_hook_func(void *curline, size_t charsz, size_t curlinelen, void *scan) +{ + char *line = (char *)curline; + resscan_line((ResScan *)scan, line, curlinelen); + return charsz * curlinelen; +} + +void httpmgr_aksk_sign(HttpMgr *http) +{ +#define KM_HOST_LEN 64 + char *start; + char host[KM_HOST_LEN] = {0}; + struct curl_slist *hdr; + size_t i; + + if (km_err_catch(http->err)) { + return; + } + + if (http->ak == NULL || http->sk == NULL) { + km_err_msg(http->err, "the ak or sk is null"); + return; + } + + if (http->url == NULL || http->reqbody == NULL) { + km_err_msg(http->err, "the http url or http request is null"); + return; + } + + /* find host from url */ + start = strstr(http->url, "://"); + if (start == NULL) { + km_err_msg(http->err, "failed to find '://' from url '%s'", http->url); + return; + } + start += strlen("://"); + for (i = 0; start[i] != '/' && start[i] != '\0'; i++) { + if (i == KM_HOST_LEN - 1) { + km_err_msg(http->err, "the host name in url '%s' is too long, maximum is 64", http->url); + return; + } + host[i] = start[i]; + } + host[i] = '\0'; + + if (strlen(host) < strlen("x.x")) { + km_err_msg(http->err, "the host name in url '%s' is too short", http->url); + return; + } + + AkSkPara para = { + http->ak, + http->sk, + + "POST", /* only sopport post for now */ + host, + start + i, /* uri */ + + {0}, /* headers, set next */ + 0, + + http->reqbody + }; + + for (hdr = http->reqhdrs; hdr != NULL; hdr = hdr->next) { + km_aksk_add_reqhdr(¶, hdr->data); + } + + AkSkSign sign = km_aksk_sign(¶); + if (sign.signature == NULL) { + km_err_msg(http->err, "failed to sign http request message of '%s'", http->url); + } else { + httpmgr_set_req_header(http, sign.host); + httpmgr_set_req_header(http, sign.date); + httpmgr_set_req_header(http, sign.signature); + } + + km_safe_free(sign.host); + km_safe_free(sign.date); + km_safe_free(sign.signature); +} + +/* + * send http request message, and store response in http->resscan. + */ +void httpmgr_receive(HttpMgr *http) +{ + if (http->ak != NULL || http->sk != NULL) { + httpmgr_aksk_sign(http); + } + + /* make sure set http request field succeed */ + if (km_err_catch(http->err)) { + return; + } + + if (http->resscan == NULL) { + km_err_msg(http->err, "please set http response receiver"); + return; + } + + CURLcode ret = curl_easy_setopt(http->client, CURLOPT_HTTPHEADER, http->reqhdrs); + CHK_CURL(http, ret); + ret = curl_easy_setopt(http->client, CURLOPT_HEADER, 1); + CHK_CURL(http, ret); + ret = curl_easy_setopt(http->client, CURLOPT_WRITEFUNCTION, http_res_hook_func); + CHK_CURL(http, ret); + ret = curl_easy_setopt(http->client, CURLOPT_WRITEDATA, http->resscan); + CHK_CURL(http, ret); + ret = curl_easy_setopt(http->client, CURLOPT_FOLLOWLOCATION, 1); + CHK_CURL(http, ret); + + /* check curl error */ + if (km_err_catch(http->err)) { + curl_slist_free_all(http->reqhdrs); + http->reqhdrs = NULL; + return; + } + +#ifdef ENABLE_KM_DEBUG + resscan_output(http->resscan, "@receive\n"); +#endif + ret = curl_easy_perform(http->client); + CHK_CURL(http, ret); + ret = curl_easy_getinfo(http->client, CURLINFO_RESPONSE_CODE, &http->state); + CHK_CURL(http, ret); + + curl_slist_free_all(http->reqhdrs); + http->reqhdrs = NULL; + http->reqbody = NULL; +#ifdef ENABLE_KM_DEBUG + resscan_finish(http->resscan); +#endif +} diff --git a/src/gausskernel/security/keymgr/comm/security_httpscan.cpp b/src/gausskernel/security/keymgr/comm/security_httpscan.cpp new file mode 100644 index 000000000..546521068 --- /dev/null +++ b/src/gausskernel/security/keymgr/comm/security_httpscan.cpp @@ -0,0 +1,281 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * security_httpscan.cpp + * + * IDENTIFICATION + * src/gausskernel/security/keymgr/src/comm/security_httpscan.cpp + * + * ------------------------------------------------------------------------- + */ + +#include "keymgr/comm/security_httpscan.h" +#include +#include +#include +#include +#include "keymgr/comm/security_utils.h" + +/* + * when we parse http response message, We judge field we are parsing based on special characters. + */ +static ResAction action_get(const char *buf, size_t bufsz) +{ + if (buf == NULL || bufsz == 0) { + return RA_NULL; + } else if (strcmp(buf, "0") == 0) { + return RA_ZERO; + } else if (strcmp(buf, "\n") == 0) { + return RA_NEWLINE; + } else if (strcmp(buf, "\r\n") == 0) { /* act of curl with diefferent version is different */ + return RA_NEWLINE; + } else if (strncmp(buf, "HTTP/", strlen("HTTP/")) == 0) { + return RA_START; + } else { + return RA_TEXT; + } +} + +#define RESBODY_BUF_SZ 4096 +/* + * remain[i] : if copy field in http response + * if 1: copy, and store in http->resscan + * if 0: do not copy, which means we will lose this field + */ +ResScan *resscan_new(KmErr *errbuf) +{ + ResScan *scan; + + scan = (ResScan *)km_alloc_zero(sizeof(ResScan)); + if (scan == NULL) { + return NULL; + } + + scan->err = errbuf; + scan->stat = RS_INIT; + + return scan; +} + +void resscan_free(ResScan *scan) +{ + if (scan == NULL) { + return; + } + + km_safe_free(scan->line); + km_safe_free(scan->header); + km_safe_free(scan->hdrkey); + km_safe_free(scan->file); + km_safe_free(scan->body); + + if (scan->fd != NULL) { + fclose(scan->fd); + scan->fd = NULL; + } + + km_free(scan); +} + +void resscan_set_receive(ResScan *scan, char remain[3], const char *hdrkey) +{ + errno_t rc; + + if (scan == NULL) { + return; + } + + scan->remain[RR_LINE] = remain[RR_LINE]; + scan->remain[RR_HEADER] = remain[RR_HEADER]; + scan->remain[RR_BODY] = remain[RR_BODY]; + + if (remain[RR_BODY] == 1) { + if (scan->body == NULL) { + scan->body = (char *)km_alloc(RESBODY_BUF_SZ); + } + if (scan->body == NULL) { + km_err_msg(scan->err, "faild to malloc memory for http response message."); + return; + } + rc = memset_s(scan->body, RESBODY_BUF_SZ, 0, RESBODY_BUF_SZ); + km_securec_check(rc, "", ""); + } + + if (scan->hdrkey != NULL) { + km_free(scan->hdrkey); + } + scan->hdrkey = km_strdup(hdrkey); +} + +#ifdef ENABLE_KM_DEBUG +void resscan_set_output(ResScan *scan, const char *file) +{ + struct stat filestat; + int fd; + + if (scan == NULL || file == NULL) { + return; + } + + scan->file = km_strdup(file); + + if (lstat(file, &filestat) < 0) { + fd = open(file, O_CREAT | O_RDWR, S_IRUSR | S_IWUSR); + if (fd >= 0) { + close(fd); + } + } + + scan->fd = fopen(scan->file, "a"); + if (scan->fd == NULL) { + km_err_msg(scan->err, "failed to open file: '%s'.", file); + } +} + +void resscan_output(ResScan *scan, const char *line) +{ + if (line == NULL) { + return; + } + if (scan->file == NULL) { + return; + } + if (scan->fd != NULL) { + (void)fputs(line, scan->fd); + } +} + +void resscan_finish(ResScan *scan) +{ + if (scan->fd != NULL) { + fclose(scan->fd); + scan->fd = NULL; + } +} +#endif + +#define RES_SCAN_ERR_MSG \ + "failed to parse http response message. we only acceept format like " \ + "'HTTP/...\\nResponseHeaders\\r\\nResponseBody\\n'. " \ + "this error may be caused by a network or proxy error." + +/* see 'state machine' */ +void resscan_line(ResScan *scan, const char *line, size_t linelen) +{ +#define RES_SCAN_ERR(scan) \ + do { \ + km_err_msg((scan)->err, RES_SCAN_ERR_MSG); \ + (scan)->stat = RS_ERROR; \ + } while (0) + + errno_t rc; + size_t catlen; + + if (scan->stat == RS_ENDEARLY) { + return; + } + + ResAction action = action_get(line, linelen); + +#ifdef ENABLE_KM_DEBUG + resscan_output(scan, line); +#endif + + switch (scan->stat) { + case RS_INIT: + if (action == RA_START) { + scan->stat = RS_LINE; + /* no break here */ + } else { + RES_SCAN_ERR(scan); + break; + } + case RS_LINE: + /* in some strange scenes, there several pieces of http response line, such as: proxy... */ + if (action == RA_START) { + if (scan->remain[RR_LINE] == 1) { + scan->line = km_strdup(line); + } + scan->stat = RS_HEADER; + break; + } + RES_SCAN_ERR(scan); + break; + case RS_HEADER: + if (action == RA_TEXT) { + if (scan->remain[RR_HEADER] != 1) { + break; + } + if (strncmp(line, scan->hdrkey, strlen(scan->hdrkey)) == 0) { + scan->header = km_strdup(line); + if (scan->remain[RR_BODY] != 1) { + scan->stat = RS_ENDEARLY; + } + } + break; + } else if (action == RA_NEWLINE) { + scan->stat = RS_BODY; + break; + } + RES_SCAN_ERR(scan); + break; + case RS_BODY: + if (action == RA_TEXT) { + if (scan->remain[RR_BODY] == 1) { + catlen = (RESBODY_BUF_SZ - scan->bodylen) - 1 > linelen ? linelen : 0; + rc = strncat_s(scan->body, RESBODY_BUF_SZ, line, catlen); + km_securec_check(rc, "", ""); + scan->bodylen += catlen; + } + break; + } else if (action == RA_NEWLINE) { + scan->stat = RS_END; + break; + } else if (action == RA_START) { + /* + * if thereis an proxy-agent, the format of http response message will be like + * HTTP/1.1 xxx xxx + * Proxy-agent: xxx + * + * HTTP/1.1 xxx xxx + */ + if (scan->remain[RR_BODY] == 1) { + rc = memset_s(scan->body, RESBODY_BUF_SZ, 0, RESBODY_BUF_SZ); + km_securec_check(rc, "", ""); + scan->bodylen = 0; + } + scan->stat = RS_LINE; + resscan_line(scan, line, linelen); + return; + } + RES_SCAN_ERR(scan); + break; + case RS_END: + if (action == RA_NEWLINE) { + /* do nothing */ + break; + } + RES_SCAN_ERR(scan); + break; + case RS_ENDEARLY: + /* ignore everything */ + break; + case RS_ERROR: + break; + default: + scan->stat = RS_ERROR; + break; + } +} diff --git a/src/gausskernel/security/keymgr/comm/security_json.cpp b/src/gausskernel/security/keymgr/comm/security_json.cpp new file mode 100644 index 000000000..ab990e18c --- /dev/null +++ b/src/gausskernel/security/keymgr/comm/security_json.cpp @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * security_json.cpp + * + * IDENTIFICATION + * src/gausskernel/security/keymgr/src/comm/security_json.cpp + * + * ------------------------------------------------------------------------- + */ + +#include "keymgr/comm/security_json.h" +#include +#include "keymgr/comm/security_error.h" +#include "keymgr/comm/security_utils.h" +#include "cjson/cJSON.h" + +static cJSON *json_format(const char *str) +{ + if (str == NULL) { + return NULL; + } + + if (str[0] == '(') { + /* -2 means: when process str format like: "(a string)", remove parentheses at the head and tail */ + return cJSON_ParseWithLength(str + 1, strlen(str) - 2); + } + + return cJSON_Parse(str); +} + +static int cjson_replace(cJSON *cjson, ReplaceRule rules[], int rulecnt) +{ + char *newval = NULL; + int ret; + int i; + + if (cjson == NULL) { + return 1; + } + + if (cJSON_IsString(cjson)) { + for (i = 0; i < rulecnt; i++) { + if (strcmp(rules[i].src, cJSON_GetStringValue(cjson)) != 0) { + continue; + } + + newval = cJSON_SetValuestring(cjson, rules[i].dest); + if (newval == NULL) { + return 0; + } + } + } + + ret = cjson_replace(cjson->next, rules, rulecnt); + if (ret == 0) { + return 0; + } + + return cjson_replace(cjson->child, rules, rulecnt); +} + +static char *cjson_find(cJSON *cjson, const char *key) +{ + char *result; + + if (cjson == NULL) { + return NULL; + } + + if (cjson->string != NULL) { + if (strcmp(cjson->string, key) == 0) { + return km_strdup(cJSON_GetStringValue(cjson)); + } + } + + result = cjson_find(cjson->next, key); + if (result != NULL) { + return result; + } + + return cjson_find(cjson->child, key); +} + +char *json_replace(const char *json, ReplaceRule rules[], int rulecnt) +{ + cJSON *cjson; + int ret; + char *result = NULL; + + if (json == NULL) { + return NULL; + } + + cjson = json_format(json); + if (cjson == NULL) { + return NULL; + } + + ret = cjson_replace(cjson, rules, rulecnt); + if (ret == 0) { + cJSON_Delete(cjson); + return NULL; + } + result = cJSON_Print(cjson); + cJSON_Delete(cjson); + + return result; +} + +char *json_find(const char *json, const char *key) +{ + cJSON *cjson; + char *result; + + if (json == NULL) { + return NULL; + } + + cjson = json_format(json); + if (cjson == NULL) { + return NULL; + } + + result = cjson_find(cjson, key); + cJSON_Delete(cjson); + + return result; +} \ No newline at end of file diff --git a/src/gausskernel/security/keymgr/comm/security_utils.cpp b/src/gausskernel/security/keymgr/comm/security_utils.cpp new file mode 100644 index 000000000..fdde8a7a4 --- /dev/null +++ b/src/gausskernel/security/keymgr/comm/security_utils.cpp @@ -0,0 +1,240 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * security_utils.cpp + * + * IDENTIFICATION + * src/gausskernel/security/keymgr/src/comm/security_utils.cpp + * + * ------------------------------------------------------------------------- + */ + +#include "keymgr/comm/security_utils.h" +#include +#include +#include +#include +#include +#include "keymgr/comm/security_error.h" + +char *km_str_strip(char *str) +{ + if (str == NULL || strlen(str) == 0) { + return NULL; + } + + while (strlen(str) > 1) { + if (str[0] == ' ') { + str++; + } else { + break; + } + } + + for (size_t i = 0; i < strlen(str); i++) { + if (str[i] == ' ') { + str[i] = '\0'; + break; + } + } + + return str; +} + +char *km_env_get(const char *env) +{ + static pthread_mutex_t envlock = PTHREAD_MUTEX_INITIALIZER; + char *value; + + if (env == NULL) { + return NULL; + } + + (void)pthread_mutex_lock(&envlock); + value = getenv(env); + (void)pthread_mutex_unlock(&envlock); + + return value; +} + +int km_str_start_with(const char *str, const char *start) +{ + if (str == NULL || strlen(str) == 0) { + return -1; + } + + while (strlen(str) > 1) { + if (str[0] == ' ') { + str++; + } else { + break; + } + } + + return strncasecmp(str, start, strlen(start)); +} + +KvScan *kv_scan_init(const char *input) +{ + KvScan *scan; + + if (input == NULL || strlen(input) == 0) { + return NULL; + } + + if (strlen(input) > KV_BUF_SZ) { + return NULL; + } + + scan = (KvScan *)km_alloc_zero(sizeof(KvScan)); + if (scan == NULL) { + return NULL; + } + + scan->input = input; + scan->inlen = (int)strlen(input); + + return scan; +} + +void kv_scan_drop(KvScan *scan) +{ + if (scan != NULL) { + km_free(scan); + } +} + +/* the kmsargs format should be 'key=value, key=value, ...' */ +int kv_scan_exec(KvScan *scan) +{ + bool isfind; + int start; + int curpos; + char curchr; + errno_t rc; + + if (scan == NULL) { + return -1; + } + + /* reset key & value buffer */ + if (scan->curpos > 0) { + rc = memset_s(scan->key, KV_BUF_SZ, 0, KV_BUF_SZ); + km_securec_check(rc, "", ""); + rc = memset_s(scan->value, KV_BUF_SZ, 0, KV_BUF_SZ); + km_securec_check(rc, "", ""); + } + + isfind = false; + start = scan->curpos; + + for (curpos = scan->curpos; curpos < scan->inlen; curpos++) { + curchr = scan->input[curpos]; + if (curchr == '=') { + rc = strncpy_s(scan->key, KV_BUF_SZ, scan->input + start, curpos - start); + start = curpos + 1; + } else if (curchr == ',') { + rc = strncpy_s(scan->value, KV_BUF_SZ, scan->input + start, curpos - start); + start = curpos + 1; + isfind = true; + } else if (curpos == scan->inlen - 1) { /* scan is complete */ + rc = strcpy_s(scan->value, KV_BUF_SZ, scan->input + start); + isfind = true; + } else { + continue; + } + km_securec_check(rc, "", ""); + + if (isfind) { + break; + } + } + + scan->curpos = curpos + 1; + if (strlen(scan->key) == 0 || strlen(scan->value) == 0) { + return -1; + } + + if (scan->curpos >= scan->inlen - 1) { + return 0; + } + + return ++scan->kvcnt; +} + +char *km_realpath(const char *path, KmErr *err) +{ + char *rpath; + struct stat statbuf; + const char dangerchar[] = {'|', ';', '&', '$', '<', '>', '`', '\\', '\'', '\"', '{', '}', '(', ')', '[', ']', '~', + '*', '?', '!'}; + + if (path == NULL || strlen(path) == 0) { + return NULL; + } + + rpath = (char *)malloc(PATH_MAX); + if (rpath == NULL) { + km_err_msg(err, "key manager failed to malloc memory."); + return NULL; + } + + if (realpath(path, rpath) == NULL) { + km_err_msg(err, "the path '%s' is not exists.", path); + km_free(rpath); + return NULL; + } + + if (lstat(rpath, &statbuf) >= 0) { + if (!S_ISREG(statbuf.st_mode) || statbuf.st_mode & (S_IRWXG | S_IRWXO) || + (statbuf.st_mode & S_IRWXU) == S_IRWXU) { + km_err_msg(err, "the permisson of '%s' should be 0600(u=rw) or less.", path); + km_free(rpath); + return NULL; + } + } + + for (size_t i = 0; i < ARR_LEN(dangerchar); i++) { + if (strchr(rpath, dangerchar[i]) != NULL) { + km_err_msg(err, "the path '%s' contain invalid character '%c',", path, dangerchar[i]); + km_free(rpath); + return NULL; + } + } + + return rpath; +} + +const char *km_str_spilt(const char *data, char *buf, size_t bufsz) +{ + const char *end; + size_t bufuse = 0; + + if (data == NULL) { + return NULL; + } + + for (end = data; end[0] != '\0' && end[0] == ' '; end++) {}; /* skip whitespace at the head */ + for (; end[0] != '\0'; end++) { + if (end[0] == ',' || (bufuse == bufsz - 1)) { + break; + } + buf[bufuse++] = end[0]; + } + for (; bufuse > 0 && buf[bufuse - 1] == ' '; bufuse--) {}; /* skip whitespace at the tail */ + buf[bufuse] = '\0'; + + return end[0] == ',' ? end + 1 : NULL; +} \ No newline at end of file diff --git a/src/common/interfaces/libpq/client_logic_hooks/encryption_hooks/aead_aes_hamc_enc_key.cpp b/src/gausskernel/security/keymgr/encrypt/security_aead_aes_hmac_enc_key.cpp old mode 100755 new mode 100644 similarity index 78% rename from src/common/interfaces/libpq/client_logic_hooks/encryption_hooks/aead_aes_hamc_enc_key.cpp rename to src/gausskernel/security/keymgr/encrypt/security_aead_aes_hmac_enc_key.cpp index 46e5c1c6f..db55850b0 --- a/src/common/interfaces/libpq/client_logic_hooks/encryption_hooks/aead_aes_hamc_enc_key.cpp +++ b/src/gausskernel/security/keymgr/encrypt/security_aead_aes_hmac_enc_key.cpp @@ -13,10 +13,10 @@ * See the Mulan PSL v2 for more details. * ------------------------------------------------------------------------- * - * aead_aes_hamc_enc_key.cpp + * security_aead_aes_hamc_enc_key.cpp * * IDENTIFICATION - * src\common\interfaces\libpq\client_logic_hooks\encryption_hooks\aead_aes_hamc_enc_key.cpp + * src/gausskernel/security/keymgr/src/encrypt/security_aead_aes_hamc_enc_key.cpp * * ------------------------------------------------------------------------- */ @@ -28,8 +28,9 @@ #include "securec.h" #include "securec_check.h" -#include "aead_aes_hamc_enc_key.h" -#include "encrypt_decrypt.h" +#include "keymgr/comm/security_error.h" +#include "keymgr/encrypt/security_aead_aes_hamc_enc_key.h" +#include "keymgr/encrypt/security_encrypt_decrypt.h" const unsigned char *AeadAesHamcEncKey::g_encryption_key_salt_format = (unsigned char *)"mppdb cell encryption key with encryption algorithm AEAD_AES_256_CBC_HMAC_SHA256 and key length " @@ -45,28 +46,49 @@ const unsigned char *AeadAesHamcEncKey::g_iv_key_salt_format = (unsigned char *)"mppdb cell IV key with encryption algorithm AEAD_AES_256_CBC_HMAC_SHA256 and key length : 32"; const int RAND_COUNT = 100; + +void HmacCtxGroup::free_hmac_ctx(HMAC_CTX** ctx_tmp) const +{ + if (*ctx_tmp != NULL) { + HMAC_CTX_free(*ctx_tmp); + *ctx_tmp = NULL; + } +} + /* Derives all the required keys from the given root key */ AeadAesHamcEncKey::AeadAesHamcEncKey(unsigned char *root_key, size_t root_key_size) { errno_t rc = EOK; + encrypt_ctx = NULL; + decrypt_ctx = NULL; generate_keys(root_key, root_key_size); rc = memset_s(root_key, root_key_size, 0, root_key_size); - securec_check_c(rc, "\0", "\0"); + km_securec_check(rc, "\0", "\0"); } AeadAesHamcEncKey::AeadAesHamcEncKey() { + encrypt_ctx = NULL; + decrypt_ctx = NULL; } AeadAesHamcEncKey::~AeadAesHamcEncKey() { errno_t rc = EOK; rc = memset_s(&_encryption_key, sizeof(_encryption_key), 0, sizeof(_encryption_key)); - securec_check_c(rc, "\0", "\0"); + km_securec_check(rc, "\0", "\0"); rc = memset_s(&_mac_key, sizeof(_mac_key), 0, sizeof(_mac_key)); - securec_check_c(rc, "\0", "\0"); + km_securec_check(rc, "\0", "\0"); rc = memset_s(&_iv_key, sizeof(_mac_key), 0, sizeof(_iv_key)); - securec_check_c(rc, "\0", "\0"); + km_securec_check(rc, "\0", "\0"); + if (encrypt_ctx != NULL) { + EVP_CIPHER_CTX_free(encrypt_ctx); + encrypt_ctx = NULL; + } + if (decrypt_ctx != NULL) { + EVP_CIPHER_CTX_free(decrypt_ctx); + decrypt_ctx = NULL; + } } bool AeadAesHamcEncKey::generate_root_key(unsigned char *key, size_t &keySize) @@ -106,20 +128,28 @@ bool AeadAesHamcEncKey::generate_keys(unsigned char *root_key, size_t root_key_l hmac_ctx_group_iv.free_hmac_ctx_all(); hmac_ctx_group_mac.free_hmac_ctx_all(); hmac_ctx_group_root.free_hmac_ctx_all(); + if (encrypt_ctx != NULL) { + EVP_CIPHER_CTX_free(encrypt_ctx); + encrypt_ctx = NULL; + } + if (decrypt_ctx != NULL) { + EVP_CIPHER_CTX_free(decrypt_ctx); + decrypt_ctx = NULL; + } if (!HKDF(root_key, root_key_len, g_encryption_key_salt_format, strlen((char *)g_encryption_key_salt_format), _encryption_key)) { rc = memset_s(root_key, root_key_len, 0, root_key_len); - securec_check_c(rc, "\0", "\0"); + km_securec_check(rc, "\0", "\0"); return false; } if (!HKDF(root_key, root_key_len, g_mac_key_salt_format, strlen((char *)g_mac_key_salt_format), _mac_key)) { rc = memset_s(root_key, root_key_len, 0, root_key_len); - securec_check_c(rc, "\0", "\0"); + km_securec_check(rc, "\0", "\0"); return false; } if (!HKDF(root_key, root_key_len, g_iv_key_salt_format, strlen((char *)g_iv_key_salt_format), _iv_key)) { rc = memset_s(root_key, root_key_len, 0, root_key_len); - securec_check_c(rc, "\0", "\0"); + km_securec_check(rc, "\0", "\0"); return false; } return true; @@ -143,7 +173,7 @@ const unsigned char *AeadAesHamcEncKey::get_iv_key() const /* * Computes a keyed hash of a given text */ -bool AeadAesHamcEncKey::HKDF(const unsigned char *key, int key_len, const unsigned char *data, int data_len, +bool AeadAesHamcEncKey::HKDF(const unsigned char *key, int key_len, const unsigned char *data, int data_len, unsigned char *result) { unsigned int result_len = MAX_SIZE; diff --git a/src/gausskernel/security/keymgr/encrypt/security_encrypt_decrypt.cpp b/src/gausskernel/security/keymgr/encrypt/security_encrypt_decrypt.cpp new file mode 100644 index 000000000..5564ae259 --- /dev/null +++ b/src/gausskernel/security/keymgr/encrypt/security_encrypt_decrypt.cpp @@ -0,0 +1,742 @@ +/* + * Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * security_encrypt_decrypt.cpp + * + * IDENTIFICATION + * src/gausskernel/security/keymgr/src/encrypt/security_encrypt_decrypt.cpp + * + * ------------------------------------------------------------------------- + */ + +#include +#include + +#include "openssl/rand.h" +#include "openssl/err.h" +#include +#include +#include "keymgr/encrypt/security_encrypt_decrypt.h" +#include "keymgr/comm/security_error.h" + +/* Key size in bytes */ +const int g_key_size = 32; + +const int g_auth_tag_size = 32; + +const int g_gcm_auth_tag_size = 16; + +/* Block size in bytes. AES uses 16 byte blocks. */ +const int g_block_size = 16; + +const int g_iv_size = 16; /* 128 bit */ + +const int g_gcm_iv_size = 12; + +const int g_algo_version_size = 4; +/* + * Minimum Length of cipher_text. This value is 4 (version byte) + 32 (authentication tag) + 16 (IV) + 16 (minimum of 1 + * block of cipher Text) + */ +static const int get_min_ciph_len(ColumnEncryptionAlgorithm algo) +{ + switch (algo) { + case ColumnEncryptionAlgorithm::AEAD_AES_256_CBC_HMAC_SHA256: + case ColumnEncryptionAlgorithm::AEAD_AES_128_CBC_HMAC_SHA256: + case ColumnEncryptionAlgorithm::SM4_SM3: + return g_algo_version_size + g_iv_size + g_auth_tag_size + g_block_size; + case ColumnEncryptionAlgorithm::AES_256_GCM_ALGO: + return g_algo_version_size + g_gcm_iv_size + g_gcm_auth_tag_size; + case ColumnEncryptionAlgorithm::AES_256_CTR_ALGO: + return g_algo_version_size + g_iv_size; + default: + return -1; + } +} + +static const unsigned char algo_version[4] = {'1', 0, 0, 0}; + +static int encrypt(EncParam *enc_param, const EVP_CIPHER *cipher, EVP_CIPHER_CTX *&ctx); +static int decrypt(EncParam *enc_param, const EVP_CIPHER *cipher, EVP_CIPHER_CTX *&ctx); +static int my_memcmp(const void *buffer1, const void *buffer2, int count); +static bool hmac_sha256_iv(AeadAesHamcEncKey &column_encryption_key, int keylen, const unsigned char *data, + int datalen, unsigned char *result); +static bool hmac_sha256_mac(AeadAesHamcEncKey &column_encryption_key, int keylen, const unsigned char *data, + int datalen, unsigned char *result); +static bool sm3(const unsigned char *data, int datalen, unsigned char *result); +static bool get_hash_by_cek_algo(ColumnEncryptionAlgorithm column_encryption_algorithm, int hmac_length, + AeadAesHamcEncKey &column_encryption_key, const unsigned char *data, unsigned char *authentication_tag); + +static const EVP_MD* get_evp_md_by_id(unsigned long ulAlgType) +{ + const EVP_MD* md = NULL; + switch (ulAlgType & 0xFFFF) { + case NID_sha256: + case NID_hmacWithSHA256: + md = EVP_sha256(); + break; + case NID_undef: + md = EVP_md_null(); + break; + default: + break; + } + return md; +} + +/* + * @Brief : bool cached_hmac() + * @Description : computes hmac of a given data block. Calls init, update, and final. + * when ctx_ptr_template is passed in, this function uses that context, and algo type and keys are ignored + * saves time for context init + * when ctx_ptr is passed in, the function saves time for new/free + * @return : success:true, failed:false. + */ +bool cached_hmac(unsigned long algo_type, const unsigned char *key, int key_len, const unsigned char *data, + int data_len, unsigned char *result, unsigned int *result_len, HmacCtxGroup *cached_ctx_group) +{ + static unsigned char result_container[EVP_MAX_MD_SIZE]; + static const unsigned char dummy_key[1] = {'\0'}; + if (cached_ctx_group == NULL) { + return false; + } + /* new a worker ctx */ + if (cached_ctx_group->ctx_worker == NULL) { + cached_ctx_group->ctx_worker = HMAC_CTX_new(); + if (cached_ctx_group->ctx_worker == NULL) { + goto err; + } + } + /* create a new template ctx */ + if (cached_ctx_group->ctx_template == NULL) { + /* making new template in local_ctx_group */ + const EVP_MD* evp_md = get_evp_md_by_id(algo_type); + cached_ctx_group->ctx_template = HMAC_CTX_new(); + if (cached_ctx_group->ctx_template == NULL) { + goto err; + } + if (key == NULL && key_len == 0) { + key = dummy_key; + } + if (!HMAC_Init_ex(cached_ctx_group->ctx_template, key, key_len, evp_md, NULL)) { + goto err; + } + } + /* copy contents of template to worker ctx */ + if (!HMAC_CTX_copy(cached_ctx_group->ctx_worker, cached_ctx_group->ctx_template)) { + goto err; + } + + /* run std lib func to calculate hmac */ + if (result == NULL) { + result = result_container; + } + if (!HMAC_Update(cached_ctx_group->ctx_worker, data, data_len)) { + goto err; + } + if (!HMAC_Final(cached_ctx_group->ctx_worker, result, result_len)) { + goto err; + } + return true; +err: + cached_ctx_group->free_hmac_ctx_all(); + return false; +} + +static const EVP_CIPHER* get_evp_cipher_md_by_algo(ColumnEncryptionAlgorithm columnEncryptionAlgorithm) +{ + const EVP_CIPHER *cipher = NULL; + switch (columnEncryptionAlgorithm) { + case ColumnEncryptionAlgorithm::AEAD_AES_256_CBC_HMAC_SHA256: + cipher = EVP_aes_256_cbc(); + break; + case ColumnEncryptionAlgorithm::AEAD_AES_128_CBC_HMAC_SHA256: + cipher = EVP_aes_128_cbc(); + break; + case ColumnEncryptionAlgorithm::SM4_SM3: + cipher = EVP_sm4_cbc(); + break; + case ColumnEncryptionAlgorithm::AES_256_GCM_ALGO: + cipher = EVP_aes_256_gcm(); + break; + case ColumnEncryptionAlgorithm::AES_256_CTR_ALGO: + cipher = EVP_aes_256_ctr(); + break; + default: + break; + } + return cipher; +} + +/* + * Computes a keyed hash of a given text + * currently used for both generating a MAC and as a KDF + */ +static bool hmac_sha256_iv(AeadAesHamcEncKey &column_encryption_key, int keylen, const unsigned char *data, + int datalen, unsigned char *result) +{ + unsigned int result_len = g_key_size; + if (!cached_hmac(NID_hmacWithSHA256, (const unsigned char *)column_encryption_key.get_iv_key(), keylen, data, + datalen, result, &result_len, &column_encryption_key.hmac_ctx_group_iv)) { + return false; + } + return true; +} + +static bool hmac_sha256_mac(AeadAesHamcEncKey &column_encryption_key, int keylen, const unsigned char *data, + int datalen, unsigned char *result) +{ + unsigned int result_len = g_key_size; + if (!cached_hmac(NID_hmacWithSHA256, (const unsigned char *)column_encryption_key.get_mac_key(), keylen, data, + datalen, result, &result_len, &column_encryption_key.hmac_ctx_group_mac)) { + return false; + } + return true; +} + +/* + * Computes a keyed hash of a given text by sm3 + */ +static bool sm3(const unsigned char *data, int datalen, unsigned char *result) +{ + unsigned int result_len = g_key_size; + EVP_MD_CTX *md_ctx = NULL; + md_ctx = EVP_MD_CTX_new(); + if (md_ctx == NULL) { + printf("ERROR(CLIENT): Fail to create the context in sm3 algorithm.\n"); + return false; + } + if (!EVP_DigestInit_ex(md_ctx, EVP_sm3(), NULL)) { + printf("ERROR(CLIENT): Fail to initialise the context in sm3 algorithm.\n"); + EVP_MD_CTX_free(md_ctx); + return false; + } + if (!EVP_DigestUpdate(md_ctx, data, (size_t)datalen)) { + printf("ERROR(CLIENT): Fail to compute digest in sm3 algorithm.\n"); + EVP_MD_CTX_free(md_ctx); + return false; + } + if (!EVP_DigestFinal_ex(md_ctx, result, &result_len)) { + printf("ERROR(CLIENT): Fail to compute digest final in sm3 algorithm.\n"); + EVP_MD_CTX_free(md_ctx); + return false; + } + EVP_MD_CTX_free(md_ctx); + return true; +} + +static bool get_hash_by_cek_algo(ColumnEncryptionAlgorithm column_encryption_algorithm, int hmac_length, + AeadAesHamcEncKey &column_encryption_key, const unsigned char *data, unsigned char *authentication_tag) +{ + bool res = false; + if (column_encryption_algorithm == ColumnEncryptionAlgorithm::SM4_SM3) { + res = sm3(data, hmac_length, authentication_tag); + } else { + res = hmac_sha256_mac(column_encryption_key, g_auth_tag_size, data, hmac_length, authentication_tag); + } + return res; +} + +/* + * To calculate the ciphertext buffer size + */ +int get_cipher_text_size(int plain_text_size) +{ + int numBlocks = plain_text_size / g_block_size + 1; + int cipher_len = numBlocks * g_block_size; + /* Output buffer size = size of VersionByte + Authentication Tag + IV + cipher Text blocks. */ + return (g_algo_version_size + g_auth_tag_size + g_iv_size + cipher_len); +} + +/* + * Set iv len, tag len, cipher_start to EncParam + */ +static bool set_enc_param(ColumnEncryptionAlgorithm algo, EncParam *enc_param, int &cipher_start) +{ + cipher_start = 0; + switch (algo) { + case ColumnEncryptionAlgorithm::AEAD_AES_256_CBC_HMAC_SHA256: + case ColumnEncryptionAlgorithm::AEAD_AES_128_CBC_HMAC_SHA256: + case ColumnEncryptionAlgorithm::SM4_SM3: + enc_param->iv_len = g_iv_size; + enc_param->tag_len = g_auth_tag_size; + cipher_start = g_algo_version_size + g_auth_tag_size + g_iv_size; + break; + case ColumnEncryptionAlgorithm::AES_256_GCM_ALGO: + enc_param->iv_len = g_gcm_iv_size; + enc_param->tag_len = g_gcm_auth_tag_size; + cipher_start = g_algo_version_size + g_gcm_auth_tag_size + g_gcm_iv_size; + break; + case ColumnEncryptionAlgorithm::AES_256_CTR_ALGO: + enc_param->iv_len = g_iv_size; + enc_param->tag_len = 0; + cipher_start = g_algo_version_size + g_iv_size; + break; + default: + return false; + } + return true; +} + +static bool is_need_hmac_check(ColumnEncryptionAlgorithm algo) +{ + if (algo == ColumnEncryptionAlgorithm::AES_256_GCM_ALGO || + algo == ColumnEncryptionAlgorithm::AES_256_CTR_ALGO) { + return false; + } + return true; +} + +/* + * Encryption data + */ +int encrypt_data(unsigned char *plain_text, int plain_text_length, AeadAesHamcEncKey &column_encryption_key, + EncryptionType encryption_type, unsigned char *result, ColumnEncryptionAlgorithm column_encryption_algorithm) +{ + bool is_null = (plain_text == NULL || plain_text_length <= 0 || encryption_type == EncryptionType::INVALID_TYPE || + result == NULL); + if (is_null) { + /* invalid input */ + return 0; + } + errno_t res = EOK; + /* Prepare EncParam */ + EncParam enc_param; + int cipher_start = 0; + if (!set_enc_param(column_encryption_algorithm, &enc_param, cipher_start)) { + return 0; + } + enc_param.plaintext = plain_text; + enc_param.plaintext_len = plain_text_length; + enc_param.tag = result; + enc_param.ciphertext = result + cipher_start; + + /* Prepare IV.IV should be 1 single block (16 bytes) */ + unsigned char _iv [g_key_size + 1] = {0}; + unsigned char iv_truncated[g_iv_size + 1] = {0}; + if (encryption_type == EncryptionType::DETERMINISTIC_TYPE) { + /* + * determenistic encryption - we create an initiailization vector based on the plaintext - to make the + * encryption CPA-secure + * HMAC_SHA_256 + */ + bool rc = hmac_sha256_iv(column_encryption_key, g_key_size, plain_text, plain_text_length, _iv); + if (!rc) { + (void)printf("ERROR(CLIENT): fail to get iv value.\n"); + return 0; + } + + /* iv is truncated to 128 bits. */ + res = memcpy_s(iv_truncated, g_iv_size + 1, _iv, g_block_size); + if (res != EOK) { + km_securec_check(res, "\0", "\0"); + printf("ERROR(CLIENT): fail to copy 128 bit iv from 256 bit iv value.\n"); + return 0; + } + } else { + if (encryption_type != EncryptionType::RANDOMIZED_TYPE) { + return 0; + } + + if (RAND_priv_bytes(iv_truncated, g_block_size) != 1) { + return 0; + } + } + /* gcm's iv len is difference, truncate it */ + enc_param.iv = iv_truncated; + if (column_encryption_algorithm == ColumnEncryptionAlgorithm::AES_256_GCM_ALGO) { + res = memset_s(iv_truncated + g_gcm_iv_size, (g_iv_size - g_gcm_iv_size) + 1, 0, + (g_iv_size - g_gcm_iv_size) + 1); + km_securec_check(res, "\0", "\0"); + } + + const EVP_CIPHER *cipher = get_evp_cipher_md_by_algo(column_encryption_algorithm); + if (cipher == NULL) { + printf("ERROR(CLIENT): invalid column encryption encryption algorithm, please check it!.\n"); + return 0; + } + const unsigned char *key = column_encryption_key.get_encyption_key(); + /* Add the ciphertext */ + int ciphertext_size = 0; + if (column_encryption_algorithm == ColumnEncryptionAlgorithm::AEAD_AES_128_CBC_HMAC_SHA256) { + /* iv is truncated to 128 bits. */ + int encrypt_key_len = g_key_size / 2 + 1; + unsigned char encrypt_key[encrypt_key_len] = {0}; + res = memcpy_s(encrypt_key, encrypt_key_len, key, g_key_size / 2); + if (res != EOK) { + printf("ERROR(CLIENT): Fail to copy 128 bit from 256 bit key value.\n"); + km_securec_check(res, "\0", "\0"); + return 0; + } + enc_param.key = encrypt_key; + enc_param.key_len = encrypt_key_len; + ciphertext_size = encrypt(&enc_param, cipher, column_encryption_key.encrypt_ctx); + res = memset_s(encrypt_key, encrypt_key_len, 0, encrypt_key_len); + if (res != EOK) { + printf("ERROR(CLIENT): Fail to clear 128 bit key value.\n"); + return 0; + } + } else { + enc_param.key = key; + enc_param.key_len = g_key_size; + ciphertext_size = encrypt(&enc_param, cipher, column_encryption_key.encrypt_ctx); + } + if (ciphertext_size < 0) { + /* failed to encrypt */ + return 0; + } + + /* add the Algorithm Version */ + res = memcpy_s(result + enc_param.tag_len, g_algo_version_size, algo_version, g_algo_version_size); + km_securec_check(res, "\0", "\0"); + + /* add the IV */ + int ivStartIndex = enc_param.tag_len + g_algo_version_size; + res = memcpy_s(result + ivStartIndex, enc_param.iv_len, iv_truncated, enc_param.iv_len); + km_securec_check(res, "\0", "\0"); + + /* add the HMAC (of versionbyte + IV + Ciphertext) */ + int hmacDataSize = g_algo_version_size + enc_param.iv_len + ciphertext_size; + if (!is_need_hmac_check(column_encryption_algorithm)) { + return (enc_param.tag_len + hmacDataSize); + } + bool hmac_res = get_hash_by_cek_algo(column_encryption_algorithm, hmacDataSize, column_encryption_key, + result + enc_param.tag_len, result); + if (!hmac_res) { + printf("ERROR(CLIENT): Fail to compute a keyed hash of a given text.\n"); + return 0; + } + + return (enc_param.tag_len + hmacDataSize); +} + +static bool check_data_integrity_by_ctr(ColumnEncryptionAlgorithm column_encryption_algorithm, + AeadAesHamcEncKey &column_encryption_key, EncParam *enc_param) +{ + if (column_encryption_algorithm == ColumnEncryptionAlgorithm::AES_256_CTR_ALGO) { + /* + * determenistic encryption - we create an initiailization vector based on the plaintext - to make the + * encryption CPA-secure + * HMAC_SHA_256 + */ + unsigned char iv_to_verify [g_key_size + 1] = {0}; + unsigned char iv_to_verify_truncated[g_iv_size + 1] = {0}; + bool rc = hmac_sha256_iv(column_encryption_key, g_key_size, enc_param->plaintext, enc_param->plaintext_len, + iv_to_verify); + if (!rc) { + (void)printf("ERROR(CLIENT): fail to get iv value.\n"); + return false; + } + errno_t res = EOK; + /* iv is truncated to 128 bits. */ + res = memcpy_s(iv_to_verify_truncated, g_iv_size + 1, iv_to_verify, g_block_size); + if (res != EOK) { + securec_check_c(res, "\0", "\0"); + printf("ERROR(CLIENT): fail to copy 128 bit iv from 256 bit iv value.\n"); + return false; + } + if (my_memcmp(enc_param->iv, iv_to_verify_truncated, enc_param->iv_len) != 0) { + printf("ERROR(CLIENT): Fail to compute a keyed hash of a given text.\n"); + return false; + } + } + return true; +} + +/* + * Decryption steps + * 1. Validate version byte + * 2. Validate Authentication tag + * 3. Decrypt the message + */ +int decrypt_data(unsigned char *cipher_text, int cipher_text_length, + AeadAesHamcEncKey &column_encryption_key, unsigned char *decryptedtext, + ColumnEncryptionAlgorithm column_encryption_algorithm) +{ + if (cipher_text == NULL || cipher_text_length <= 0 || decryptedtext == NULL) { + return 0; + } + + if (cipher_text_length < get_min_ciph_len(column_encryption_algorithm)) { + printf("ERROR(CLIENT): The length of cipher_text is invalid, cannot decrypt.\n"); + return 0; + } + /* Prepare EncParam */ + EncParam enc_param; + int cipher_start_index = 0; // this is where cipher starts. + if (!set_enc_param(column_encryption_algorithm, &enc_param, cipher_start_index)) { + return 0; + } + enc_param.plaintext = decryptedtext; + enc_param.ciphertext_len = cipher_text_length - cipher_start_index; + enc_param.ciphertext = cipher_text + cipher_start_index; + + if (cipher_text[enc_param.tag_len] != '1') { + /* Cipher text was computed with a different algorithm version than this. */ + printf("ERROR(CLIENT): Version byte of cipher_text is invalid, cannot decrypt.\n"); + return 0; + } + unsigned char iv[enc_param.iv_len] = {0}; + enc_param.iv = iv; + errno_t rc = memcpy_s(iv, enc_param.iv_len, cipher_text + enc_param.tag_len + g_algo_version_size, + enc_param.iv_len); + km_securec_check(rc, "\0", "\0"); + + /* Computing the authentication tag */ + unsigned char authenticationTag[enc_param.tag_len + 1] = {0}; + if (column_encryption_algorithm == ColumnEncryptionAlgorithm::AES_256_CTR_ALGO) { + enc_param.tag = NULL; + } else if (column_encryption_algorithm != ColumnEncryptionAlgorithm::AES_256_GCM_ALGO) { + enc_param.tag = authenticationTag; + int HMAC_length = cipher_text_length - enc_param.tag_len; + bool res = get_hash_by_cek_algo(column_encryption_algorithm, HMAC_length, column_encryption_key, + cipher_text + enc_param.tag_len, authenticationTag); + if (!res) { + printf("ERROR(CLIENT): Fail to compute a keyed hash of a given text.\n"); + return 0; + } + + int cmp_result = my_memcmp(authenticationTag, cipher_text, enc_param.tag_len); + if (cmp_result != 0) { + /* MAC check failed */ + return 0; + } + } else { + enc_param.tag = authenticationTag; + rc = memcpy_s(authenticationTag, enc_param.tag_len, cipher_text, enc_param.tag_len); + km_securec_check(rc, "\0", "\0"); + } + + /* Decrypt the ciphertext */ + const EVP_CIPHER *cipher = get_evp_cipher_md_by_algo(column_encryption_algorithm); + if (cipher == NULL) { + printf("ERROR(CLIENT): invalid column encryption encryption algorithm, please check it!.\n"); + return 0; + } + const unsigned char *key = column_encryption_key.get_encyption_key(); + int decryptedtext_len = 0; + if (column_encryption_algorithm == ColumnEncryptionAlgorithm::AEAD_AES_128_CBC_HMAC_SHA256) { + /* iv is truncated to 128 bits. */ + int encrypt_key_len = g_key_size / 2 + 1; + unsigned char encrypt_key[encrypt_key_len] = {0}; + errno_t result = memcpy_s(encrypt_key, encrypt_key_len, key, g_key_size / 2); + if (result != EOK) { + printf("ERROR(CLIENT): Fail to copy 128 bit from 256 bit key value.\n"); + km_securec_check(result, "\0", "\0"); + return 0; + } + enc_param.key = encrypt_key; + enc_param.key_len = encrypt_key_len; + decryptedtext_len = decrypt(&enc_param, cipher, column_encryption_key.decrypt_ctx); + result = memset_s(encrypt_key, encrypt_key_len, 0, encrypt_key_len); + if (result != EOK) { + printf("ERROR(CLIENT): Fail to clear 128 bit key value.\n"); + return 0; + } + } else { + enc_param.key = key; + enc_param.key_len = g_key_size; + decryptedtext_len = decrypt(&enc_param, cipher, column_encryption_key.decrypt_ctx); + } + if (decryptedtext_len < 0) { + return 0; + } + if (!check_data_integrity_by_ctr(column_encryption_algorithm, column_encryption_key, &enc_param)) { + return 0; + } + /* Add a NULL terminator. We are expecting printable text */ + decryptedtext[decryptedtext_len] = '\0'; + return decryptedtext_len; +} + +/* + * encrypt plaintext thought AES_256_cbc or AES_256_gcm algorithm + * cell_ciphertext = AES-CBC-256(enc_key, cell_iv, cell_data) with PKCS7 padding. + */ +static int encrypt(EncParam *enc_param, const EVP_CIPHER *cipher, EVP_CIPHER_CTX *&ctx) +{ + bool is_invalid = enc_param->plaintext == NULL || enc_param->plaintext_len <= 0 || enc_param->key == NULL || + enc_param->iv == NULL || enc_param->ciphertext == NULL; + if (is_invalid) { + return 0; + } + if (ctx == NULL) { + /* Create and initialise the context */ + ctx = EVP_CIPHER_CTX_new(); + if (ctx == NULL) { + printf("ERROR(CLIENT): Fail to create and initialise the context.\n"); + return -1; + } + /* + * Initialise the encryption operation. IMPORTANT - ensure you use a key + * and IV size appropriate for your cipher + * In this example we are using 256 bit AES (i.e. a 256 bit key). The + * IV size for *most* modes is the same as the block size. For AES this + * is 128 bits + */ + if (EVP_EncryptInit_ex(ctx, cipher, NULL, NULL, NULL) != 1) { + printf("ERROR(CLIENT): Fail to create new cipher.\n"); + EVP_CIPHER_CTX_free(ctx); + ctx = NULL; + return -1; + } + + if (EVP_EncryptInit_ex(ctx, NULL, NULL, enc_param->key, enc_param->iv) != 1) { + (void)printf("ERROR(CLIENT): Fail to set key and iv.\n"); + EVP_CIPHER_CTX_free(ctx); + ctx = NULL; + return -1; + } + } else { + if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, enc_param->iv) != 1) { + (void)printf("ERROR(CLIENT): Fail to set iv.\n"); + EVP_CIPHER_CTX_free(ctx); + ctx = NULL; + return -1; + } + } + + /* + * Provide the message to be encrypted, and obtain the encrypted output. + * EVP_EncryptUpdate can be called multiple times if necessary + */ + int len = 0; + int ciphertext_len = 0; + if (EVP_EncryptUpdate(ctx, enc_param->ciphertext, &len, enc_param->plaintext, enc_param->plaintext_len) != 1) { + printf("ERROR(CLIENT): Fail to encrypt.\n"); + EVP_CIPHER_CTX_free(ctx); + ctx = NULL; + return -1; + } + ciphertext_len = len; + /* + * Finalise the encryption. Further ciphertext bytes may be written at + * this stage. + */ + if (EVP_EncryptFinal_ex(ctx, enc_param->ciphertext + len, &len) != 1) { + printf("ERROR(CLIENT): Fail to encrypt final.\n"); + EVP_CIPHER_CTX_free(ctx); + ctx = NULL; + return -1; + } + ciphertext_len += len; + if (cipher == EVP_aes_256_gcm()) { + if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_GET_TAG, enc_param->tag_len, enc_param->tag) != 1) { + (void)printf("ERROR(CLIENT): Fail to get tag during encrypt data.\n"); + EVP_CIPHER_CTX_free(ctx); + ctx = NULL; + return -1; + } + } + enc_param->ciphertext_len = ciphertext_len; + + return ciphertext_len; +} + +/* + * Decrypt the message though aes_256_cbc or AES_256_gcm algorithm + */ +static int decrypt(EncParam *enc_param, const EVP_CIPHER *cipher, EVP_CIPHER_CTX *&ctx) +{ + if (ctx == NULL) { + ctx = EVP_CIPHER_CTX_new(); + /* Create and initialise the context */ + if (ctx == NULL) { + printf("ERROR(CLIENT): keymanage cannot new ctx.\n"); + return -1; + } + /* + * Initialise the decryption operation. IMPORTANT - ensure you use a key + * and IV size appropriate for your cipher + * In this example we are using 256 bit AES (i.e. a 256 bit key). The + * IV size for *most* modes is the same as the block size. For AES this + * is 128 bits + */ + if (EVP_DecryptInit_ex(ctx, cipher, NULL, NULL, NULL) != 1) { + printf("ERROR(CLIENT): cannot create new cipher.\n"); + EVP_CIPHER_CTX_free(ctx); + ctx = NULL; + return -1; + } + + /* Initialize key and iv, in gcm, not necessary to set iv length if this is 12 bytes (96 bits) */ + if (EVP_DecryptInit_ex(ctx, NULL, NULL, enc_param->key, enc_param->iv) != 1) { + (void)printf("ERROR(CLIENT): EVP_DecryptInit_ex set key and iv error.\n"); + EVP_CIPHER_CTX_free(ctx); + ctx = NULL; + return -1; + } + } else { + if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, enc_param->iv) != 1) { + printf("ERROR(CLIENT): cannot create new cipher.\n"); + EVP_CIPHER_CTX_free(ctx); + ctx = NULL; + return -1; + } + } + + /* + * Provide the message to be decrypted, and obtain the plaintext output. + * EVP_DecryptUpdate can be called multiple times if necessary. + */ + int len = 0; + int plaintext_len = 0; + if (EVP_DecryptUpdate(ctx, enc_param->plaintext, &len, enc_param->ciphertext, enc_param->ciphertext_len) != 1) { + printf("ERROR(CLIENT): cannot EVP_EncryptUpdate.\n"); + EVP_CIPHER_CTX_free(ctx); + ctx = NULL; + return -1; + } + plaintext_len = len; + + if (cipher == EVP_aes_256_gcm()) { + if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_TAG, enc_param->tag_len, enc_param->tag) != 1) { + (void)printf("ERROR(CLIENT): EVP_CIPHER_CTX_ctrl set tag error.\n"); + EVP_CIPHER_CTX_free(ctx); + ctx = NULL; + return -1; + } + } + + /* + * Finalise the decryption. Further plaintext bytes may be written at + * this stage. + */ + if (EVP_DecryptFinal_ex(ctx, enc_param->plaintext + len, &len) != 1) { + (void)printf( + "ERROR(CLIENT): cannot EVP_DecryptFinal_ex. error: %s.\n", ERR_error_string(ERR_get_error(), NULL)); + EVP_CIPHER_CTX_free(ctx); + ctx = NULL; + return -1; + } + plaintext_len += len; + enc_param->plaintext_len = plaintext_len; + return enc_param->plaintext_len; +} + +static int my_memcmp(const void *buffer1, const void *buffer2, int count) +{ + if (!count) { + return 0; + } + while (count--) { + if (*(char *)buffer1 != *(char *)buffer2) { + return (*((unsigned char *)buffer1) - *((unsigned char *)buffer2)); + } + buffer1 = (char *)buffer1 + 1; + buffer2 = (char *)buffer2 + 1; + } + return 0; +} diff --git a/src/common/interfaces/libpq/client_logic_hooks/encryption_hooks/sm2_enc_key.cpp b/src/gausskernel/security/keymgr/encrypt/security_sm2_enc_key.cpp similarity index 90% rename from src/common/interfaces/libpq/client_logic_hooks/encryption_hooks/sm2_enc_key.cpp rename to src/gausskernel/security/keymgr/encrypt/security_sm2_enc_key.cpp index 24731dc4a..350e9898d 100644 --- a/src/common/interfaces/libpq/client_logic_hooks/encryption_hooks/sm2_enc_key.cpp +++ b/src/gausskernel/security/keymgr/encrypt/security_sm2_enc_key.cpp @@ -13,10 +13,10 @@ * See the Mulan PSL v2 for more details. * ------------------------------------------------------------------------- * - * sm2_enc_key.cpp + * security_sm2_enc_key.cpp * * IDENTIFICATION - * src\common\interfaces\libpq\client_logic_hooks\encryption_hooks\sm2_enc_key.cpp + * src/gausskernel/security/keymgr/src/encrypt/security_sm2_enc_key.cpp * * ------------------------------------------------------------------------- */ @@ -31,12 +31,11 @@ #include "openssl/ec.h" #include "openssl/bn.h" #include "openssl/crypto.h" -#include "openssl/opensslconf.h" #include "openssl/err.h" #include #include -#include "sm2_enc_key.h" -#include "libpq-fe.h" +#include "keymgr/encrypt/security_sm2_enc_key.h" +#include "keymgr/comm/security_error.h" Sm2KeyPair* generate_encrypt_pair_key() { @@ -66,7 +65,16 @@ Sm2KeyPair* generate_encrypt_pair_key() } BIO *bio_priv_key = BIO_new(BIO_s_mem()); + if (bio_priv_key == NULL) { + EC_KEY_free(encrypted_key); + return NULL; + } BIO *bio_pub_key = BIO_new(BIO_s_mem()); + if (bio_priv_key == NULL) { + BIO_free(bio_priv_key); + EC_KEY_free(encrypted_key); + return NULL; + } PEM_write_bio_ECPrivateKey(bio_priv_key, encrypted_key, NULL, NULL, 0, NULL, NULL); PEM_write_bio_EC_PUBKEY(bio_pub_key, encrypted_key); @@ -91,7 +99,7 @@ Sm2KeyPair* generate_encrypt_pair_key() priv_key->ustr_len = pri_key_len; pub_key->ustr_len = pub_key_len; - sm2_key_pair = (Sm2KeyPair *)malloc(sizeof(Sm2KeyPair)); + sm2_key_pair = (Sm2KeyPair *)km_alloc(sizeof(Sm2KeyPair)); if (sm2_key_pair == NULL) { free_cmkem_ustr(priv_key); free_cmkem_ustr(pub_key); @@ -122,11 +130,11 @@ CmkemErrCode encrypt_with_sm2_pubkey(CmkemUStr *plain, CmkemUStr *pub_key, Cmkem } bio_pub_key_len = BIO_write(bio_pub_key, pub_key->ustr_val, (int)pub_key->ustr_len); - if (bio_pub_key_len < 0) { + if (bio_pub_key_len == 0) { cmkem_errmsg("failed to write sm2 key to from BIO."); BIO_free(bio_pub_key); rc = memset_s(pub_key->ustr_val, pub_key->ustr_len, 0, pub_key->ustr_len); - securec_check_c(rc, "", ""); + km_securec_check(rc, "", ""); return CMKEM_WRITE_TO_BIO_ERR; } @@ -153,13 +161,6 @@ CmkemErrCode encrypt_with_sm2_pubkey(CmkemUStr *plain, CmkemUStr *pub_key, Cmkem return CMKEM_EVP_ERR; } - ret = EVP_PKEY_set_alias_type(public_evp_key, EVP_PKEY_SM2); - if (ret != 1) { - cmkem_errmsg("EVP_PKEY_set_alias_type to EVP_PKEY_SM2 failed!"); - EVP_PKEY_free(public_evp_key); - return CMKEM_EVP_ERR; - } - /* do cipher. */ ctx = EVP_PKEY_CTX_new(public_evp_key, NULL); EVP_PKEY_free(public_evp_key); @@ -243,19 +244,11 @@ CmkemErrCode decrypt_with_sm2_privkey(CmkemUStr *cipher, CmkemUStr *priv_key, Cm return CMKEM_EVP_ERR; } - ret = EVP_PKEY_set_alias_type(private_evp_key, EVP_PKEY_SM2); - if (ret != 1) { - cmkem_errmsg("EVP_PKEY_set_alias_type to EVP_PKEY_SM2 failed!"); - EVP_PKEY_free(private_evp_key); - return CMKEM_EVP_ERR; - } - /* do cipher. */ ctx = EVP_PKEY_CTX_new(private_evp_key, NULL); EVP_PKEY_free(private_evp_key); if (ctx == NULL) { cmkem_errmsg("EVP_PKEY_CTX_new failed"); - EVP_PKEY_free(private_evp_key); return CMKEM_MALLOC_MEM_ERR; } @@ -268,6 +261,7 @@ CmkemErrCode decrypt_with_sm2_privkey(CmkemUStr *cipher, CmkemUStr *priv_key, Cm *plain = malloc_cmkem_ustr(cipher->ustr_len * 2); if (*plain == NULL) { + EVP_PKEY_CTX_free(ctx); return CMKEM_MALLOC_MEM_ERR; } diff --git a/src/gausskernel/security/keymgr/his/security_his.cpp b/src/gausskernel/security/keymgr/his/security_his.cpp new file mode 100644 index 000000000..7f8cfd72c --- /dev/null +++ b/src/gausskernel/security/keymgr/his/security_his.cpp @@ -0,0 +1,217 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * security_his.cpp + * + * IDENTIFICATION + * src/gausskernel/security/keymgr/src/his/security_his.cpp + * + * ------------------------------------------------------------------------- + */ +#include "keymgr/his/security_his.h" +#include +#include +#include +#include "keymgr/security_key_mgr.h" +#include "keymgr/his/security_his_kms.h" +#include "keymgr/his/security_his_iam.h" + +HisMgr *his_mgr_new(KmErr *err) +{ + HisMgr *his; + + his = (HisMgr *)km_alloc_zero(sizeof(HisMgr)); + if (his == NULL) { + return NULL; + } + + his->kmgr.err = err; + his->iammgr = his_iam_new(err); + his->kmsmgr = his_kms_new(his->iammgr, err); + + if (his->iammgr == NULL || his->kmsmgr == NULL) { + his_mgr_free(his); + return NULL; + } + + return his; +} + +void his_mgr_read_env(HisMgr *his) +{ + char *env; + KvScan *scan; + int kvcnt; + + if (his == NULL) { + return; + } + + if (his->setarg == 1) { + return; + } + + env = km_env_get("HIS_KMS_INFO"); + if (env == NULL || strlen(env) == 0) { + km_err_msg(his->kmgr.err, + "please use environment variabl '$HIS_KMS_INFO' to set identity and project information of kms."); + return; + } + + scan = kv_scan_init(env); + kvcnt = kv_scan_exec(scan); + for (;;) { + if (kvcnt > 0) { + his_mgr_set_arg(his, km_str_strip(scan->key), km_str_strip(scan->value)); + kvcnt = kv_scan_exec(scan); + } else if (kvcnt == 0) { + his_mgr_set_arg(his, km_str_strip(scan->key), km_str_strip(scan->value)); + break; + } else { + km_err_msg(his->kmgr.err, "parse key value error, error at position %d of '$HIS_KMS_INFO'.", scan->curpos); + break; + } + } + kv_scan_drop(scan); +} + +void his_mgr_free(HisMgr *his) +{ + if (his == NULL) { + return; + } + + his_iam_free(his->iammgr); + his_kms_free(his->kmsmgr); + + km_free(his); +} + +void his_mgr_set_arg(HisMgr *his, const char *key, const char *value) +{ +#define MAX_ARG_LEN 512 + + if (his == NULL) { + return; + } + if (strlen(key) > MAX_ARG_LEN) { + return; + } + if (strlen(value) > MAX_ARG_LEN) { + return; + } + + his->setarg = 1; + his_iam_set_arg(his->iammgr, key, value); + his_kms_set_arg(his->kmsmgr, key, value); +} + +static KeyMgr *his_new(KmErr *err) +{ + return (KeyMgr *)his_mgr_new(err); +} + +static void his_free(KeyMgr *kmgr) +{ + his_mgr_free((HisMgr *)(void *)kmgr); +} + +static void his_set_arg(KeyMgr *kmgr, const char *key, const char *value) +{ + his_mgr_set_arg((HisMgr *)(void *)kmgr, key, value); +} + +static char *his_mk_select(KeyMgr *kmgr, KeyInfo key) +{ + HisMgr *his = (HisMgr *)(void *)kmgr; + + if (key.algo == NULL) { + km_err_msg(his->kmgr.err, "please set key algorithm type."); + return NULL; + } + + if (strcasecmp(key.algo, "aes_256") != 0) { + km_err_msg(his->kmgr.err, "invalid algorithm '%s', his_kms only support 'aes_256'.", key.algo); + return NULL; + } + + his_mgr_read_env(his); + if (km_err_catch(his->kmgr.err)) { + return NULL; + } + + return his_kms_mk_select(his->kmsmgr, key.id); +} + +static KmUnStr his_mk_encrypt(KeyMgr *kmgr, KeyInfo key, KmUnStr plain) +{ +#define HIS_KMS_BLOCK_LEN 16 + HisMgr *his = (HisMgr *)(void *)kmgr; + KmUnStr cipher = {0}; + + if (plain.len % HIS_KMS_BLOCK_LEN != 0) { + km_err_msg(his->kmgr.err, "the length of plain we sent to his_kms should be exactly divided by 16."); + return cipher; + } + + his_mgr_read_env(his); + if (km_err_catch(his->kmgr.err)) { + return cipher; + } + + return his_kms_mk_encrypt(his->kmsmgr, key.id, plain); +} + +static KmUnStr his_mk_decrypt(KeyMgr *kmgr, KeyInfo key, KmUnStr cipher) +{ + HisMgr *his = (HisMgr *)(void *)kmgr; + KmUnStr plain = {0}; + + his_mgr_read_env(his); + if (km_err_catch(his->kmgr.err)) { + return plain; + } + + return his_kms_mk_decrypt(his->kmsmgr, key.id, cipher); +} + +static KmUnStr his_dk_create(KeyMgr *kmgr, KeyInfo key, KmUnStr *cipher) +{ + HisMgr *his = (HisMgr *)(void *)kmgr; + KmUnStr plain = {0}; + + his_mgr_read_env(his); + if (km_err_catch(his->kmgr.err)) { + return plain; + } + + return his_kms_dk_create(his->kmsmgr, key.id, cipher); +} + +KeyMethod his_kms = { + "his_kms", + + his_new, /* kmgr_new */ + his_free, /* kmgr_free */ + his_set_arg, /* kmgr_set_arg */ + + NULL, /* mk_create */ + NULL, /* mk_delete */ + his_mk_select, /* mk_select */ + his_mk_encrypt, /* mk_encrypt */ + his_mk_decrypt, /* mk_decrypt */ + + his_dk_create, /* dk_create */ +}; \ No newline at end of file diff --git a/src/gausskernel/security/keymgr/his/security_his_iam.cpp b/src/gausskernel/security/keymgr/his/security_his_iam.cpp new file mode 100644 index 000000000..c130e6a58 --- /dev/null +++ b/src/gausskernel/security/keymgr/his/security_his_iam.cpp @@ -0,0 +1,298 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * security_his_iam.cpp + * + * IDENTIFICATION + * src/gausskernel/security/keymgr/src/his/security_his_iam.cpp + * + * ------------------------------------------------------------------------- + */ + +#include "keymgr/his/security_his_iam.h" + +#include +#include + +#include "keymgr/comm/security_http.h" +#include "keymgr/comm/security_utils.h" +#include "keymgr/comm/security_json.h" + +HisIamMgr *his_iam_new(KmErr *errbuf) +{ + HisIamMgr *iam; + + iam = (HisIamMgr *)km_alloc_zero(sizeof(HisIamMgr)); + if (iam == NULL) { + return NULL; + } + + iam->err = errbuf; + + iam->httpmgr = httpmgr_new(errbuf); + if (iam->httpmgr == NULL) { + km_free(iam); + return NULL; + } + + return iam; +} + +void his_iam_free(HisIamMgr *iam) +{ + errno_t rc; + + if (iam == NULL) { + return; + } + + km_safe_free(iam->account); + if (iam->secret != NULL) { + rc = memset_s(iam->secret, strlen(iam->secret), 0, strlen(iam->secret)); + km_securec_check(rc, "", ""); + km_free(iam->secret); + } + km_safe_free(iam->appid); + km_safe_free(iam->enterprise); + + km_safe_free(iam->url); + + httpmgr_free(iam->httpmgr); + km_safe_free(iam->cacert); + + km_safe_free(iam->reqbody); + if (iam->token != NULL) { + rc = memset_s(iam->token, strlen(iam->token), 0, strlen(iam->token)); + km_securec_check(rc, "", ""); + km_free(iam->token); + } + + km_free(iam); +} + +/* + * set iam authentication info, and cache them. + * when the token expired, we will use cached info to refresh token. + */ +void his_iam_set_arg(HisIamMgr *iam, const char *key, const char *value) +{ + errno_t rc = 0; + + if (key == NULL || value == NULL) { + return; + } + + /* + * if we have cached the http reqbody and token. + * when we change any parameters, we need to clear the cache at the same time. + */ + km_safe_free(iam->reqbody); + if (iam->token != NULL) { + rc = memset_s(iam->token, strlen(iam->token), 0, strlen(iam->token)); + km_securec_check(rc, "", ""); + km_free(iam->token); + iam->token = NULL; + } + + if (strcasecmp(key, "hisaccount") == 0) { + km_safe_free(iam->account); /* free the old one if need */ + iam->account = km_strdup(value); + } else if (strcasecmp(key, "hissecret") == 0) { + if (iam->secret != NULL) { + rc = memset_s(iam->secret, strlen(iam->secret), 0, strlen(iam->secret)); + km_securec_check(rc, "", ""); + km_free(iam->secret); + } + iam->secret = km_strdup(value); + } else if (strcasecmp(key, "hisappid") == 0) { + km_safe_free(iam->appid); + iam->appid = km_strdup(value); + } else if (strcasecmp(key, "hisenterprise") == 0) { + km_safe_free(iam->enterprise); + iam->enterprise = km_strdup(value); + } else if (strcasecmp(key, "hisiamurl") == 0) { + km_safe_free(iam->url); + iam->url = km_strdup(value); + } /* ignore unknowned para */ +} + +static void his_iam_has_set_auth_info(HisIamMgr *iam) +{ + if (iam->account == NULL) { + km_err_msg(iam->err, "failed to access his iam service, plase set parameter 'hisAccount'"); + return; + } + if (iam->secret == NULL) { + km_err_msg(iam->err, "failed to access his iam service, plase set parameter 'hisSecret'"); + return; + } + if (iam->appid == NULL) { + km_err_msg(iam->err, "failed to access his iam service, plase set parameter 'hisAppid'"); + return; + } + if (iam->enterprise == NULL) { + km_err_msg(iam->err, "failed to access his iam service, plase set parameter 'hisEnterprise'"); + return; + } +} + +static char *his_iam_get_url(HisIamMgr *iam) +{ +#define HIS_IAM_URL_LEN 256 + if (iam->url == NULL) { + km_err_msg(iam->err, "failed to access his iam service, plase set parameter 'hisIamUrl'"); + return NULL; + } + + if (strlen(iam->url) > HIS_IAM_URL_LEN) { + km_err_msg(iam->err, "failed to access his iam service, the length of 'iamUrl' exceeds 256"); + return NULL; + } + + return iam->url; +} + +static char *his_iam_get_reqbody(HisIamMgr *iam) +{ + his_iam_has_set_auth_info(iam); + if (km_err_catch(iam->err)) { + return NULL; + } + + const char *reqtemp = JS_TO_STR(( + { + "data": { + "type": "JWT-Token", + "attributes": { + "method": "CREATE", + "account": "{account}", + "secret": "{secret}", + "project": "{appid}", + "enterprise": "{enterprise}" + } + } + } + )); + + ReplaceRule rules[] = { + {"{account}", iam->account}, + {"{secret}", iam->secret}, + {"{appid}", iam->appid}, + {"{enterprise}", iam->enterprise} + }; + km_safe_free(iam->reqbody); + iam->reqbody = json_replace(reqtemp, rules, ARR_LEN(rules)); + if (iam->reqbody == NULL) { + km_err_msg(iam->err, "when create iam request body, failed to parse json template"); + } + + return iam->reqbody; +} + +static char *his_iam_parse_token(HisIamMgr *iam) +{ +#define MAX_TOKEN_LEN 2048 + char *resbody; + char *token; + long state; + errno_t rc; + size_t tkhdrlen; + + resbody = httpmgr_get_res_body(iam->httpmgr); + state = httpmgr_get_status(iam->httpmgr); + if (state >= HTTP_MIN_UNEXCEPTED_CODE) { + km_err_msg(iam->err, "failed to access '%s', http status: %ld, error message: %s", iam->url, state, resbody); + return NULL; + } + + token = json_find(resbody, "access_token"); + if (token == NULL) { + km_err_msg(iam->err, "when access '%s', failed find 'access_token' from http response body", iam->url); + return NULL; + } + + /* we cache token until it gets expired */ + tkhdrlen = strlen("Authorization: ") + strlen(token) + 1; + if (tkhdrlen > MAX_TOKEN_LEN) { + rc = memset_s(token, strlen(token), 0, strlen(token)); + km_securec_check(rc, "", ""); + km_free(token); + + km_err_msg(iam->err, "invalid token header len: %lu", tkhdrlen); + return NULL; + } + + iam->token = (char *)km_alloc(tkhdrlen); + if (iam->token == NULL) { + rc = memset_s(token, strlen(token), 0, strlen(token)); + km_securec_check(rc, "", ""); + km_free(token); + + km_err_msg(iam->err, "malloc memory error"); + return NULL; + } + + rc = sprintf_s(iam->token, tkhdrlen, "Authorization: %s", token); + km_securec_check_ss(rc, "", ""); + + rc = memset_s(token, strlen(token), 0, strlen(token)); + km_securec_check(rc, "", ""); + km_free(token); + + return iam->token; +} + +char *his_iam_get_token(HisIamMgr *iam) +{ + HttpMgr *http; + + /* if we have cache token, just return it. */ + if (iam->token != NULL) { + return iam->token; + } + + http = iam->httpmgr; +#ifdef ENABLE_KM_DEBUG + httpmgr_set_output(http, "./iam.out"); +#endif + httpmgr_set_req_line(http, his_iam_get_url(iam), CURLOPT_HTTPPOST, iam->cacert); + httpmgr_set_req_header(http, "Content-Type:application/json"); + httpmgr_set_req_body(http, his_iam_get_reqbody(iam)); + char remain[3] = {0, 0, 1}; + httpmgr_set_response(http, remain, NULL); + + httpmgr_receive(http); + /* check client error here */ + if (km_err_catch(iam->err)) { + return NULL; + } + + /* check server error here, such as http 400, http 500 */ + return his_iam_parse_token(iam); +} + +char *his_iam_refresh_token(HisIamMgr *iam) +{ + errno_t rc; + + if (iam->token != NULL) { + rc = memset_s(iam->token, strlen(iam->token), 0, strlen(iam->token)); + km_securec_check(rc, "", ""); + km_free(iam->token); + iam->token = NULL; + } + + return his_iam_get_token(iam); +} \ No newline at end of file diff --git a/src/gausskernel/security/keymgr/his/security_his_kms.cpp b/src/gausskernel/security/keymgr/his/security_his_kms.cpp new file mode 100644 index 000000000..420d415be --- /dev/null +++ b/src/gausskernel/security/keymgr/his/security_his_kms.cpp @@ -0,0 +1,528 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * security_his_kms.cpp + * + * IDENTIFICATION + * src/gausskernel/security/keymgr/src/his/security_his_kms.cpp + * + * ------------------------------------------------------------------------- + */ + +#include "keymgr/his/security_his_kms.h" +#include +#include +#include "keymgr/comm/security_http.h" +#include "keymgr/comm/security_utils.h" +#include "keymgr/comm/security_json.h" +#include "keymgr/comm/security_encode.h" +#include "keymgr/his/security_his_iam.h" + +typedef struct { + char *kmsurl; + char *keyid; +} HisKmsKeyPath; + +HisKmsMgr *his_kms_new(HisIamMgr *iam, KmErr *errbuf) +{ + HisKmsMgr *kms; + + if (iam == NULL) { + return NULL; + } + + kms = (HisKmsMgr *)km_alloc_zero(sizeof(HisKmsMgr)); + if (kms == NULL) { + return NULL; + } + + kms->err = errbuf; + kms->iam = iam; + kms->httpmgr = httpmgr_new(errbuf); + if (kms->httpmgr == NULL) { + km_free(kms); + return NULL; + } + + return kms; +} + +void his_kms_free(HisKmsMgr *kms) +{ + if (kms == NULL) { + return; + } + + km_safe_free(kms->kmsurl); + km_safe_free(kms->appid); + km_safe_free(kms->domain); + km_safe_free(kms->reqbody); + + httpmgr_free(kms->httpmgr); + km_safe_free(kms->cacert); + + km_free(kms); +} + +void his_kms_set_arg(HisKmsMgr *kms, const char *key, const char *value) +{ + if (key == NULL || value == NULL) { + return; + } + + if (strcasecmp(key, "hisappid") == 0) { + km_safe_free(kms->appid); + kms->appid = km_strdup(value); + } else if (strcasecmp(key, "hisdomain") == 0) { + km_safe_free(kms->domain); + kms->domain = km_strdup(value); + } else if (strcasecmp(key, "hiskmsurl") == 0) { + km_safe_free(kms->kmsurl); + kms->kmsurl = km_strdup(value); + } else if (strcasecmp(key, "hiscacert") == 0) { + km_safe_free(kms->cacert); + kms->cacert = km_realpath(value, kms->err); + } /* else just ignore unkown para */ +} + +/* the format of keypath is 'URL/ID', we need return 'KEYID' */ +static HisKmsKeyPath his_kms_parser_key_path(HisKmsMgr *kms, const char *keypath) +{ +#define HIS_KMS_URL_LEN 1024 + size_t split = 0; + HisKmsKeyPath kpath = {0}; + size_t i; + + if (keypath == NULL || strlen(keypath) <= 1) { + km_err_msg(kms->err, "his kms failed to get keypath filed."); + return kpath; + } + + if (strlen(keypath) > HIS_KMS_URL_LEN) { + km_err_msg(kms->err, "failed to access his kms, the length of keypath exceeds 1024."); + return kpath; + } + + for (i = strlen(keypath) - 1; i > 0; i--) { + if (keypath[i] == '/') { + split = i; + break; + } + } + + if (split == 0 || split == strlen(keypath) - 1) { + km_err_msg(kms->err, "the format of keypath should be like 'KmsUrl/KeyId', but not '%s'.", keypath); + return kpath; + } + + kpath.keyid = km_strdup(keypath + split + 1); + kpath.kmsurl = km_strndup(keypath, split); + + return kpath; +} + +typedef enum { + SEL_MK = 0, + ENC_KEY, + DEC_KEY +} UrlType; + +static char *his_kms_get_url(HisKmsMgr *kms, const char *kmsurl, UrlType type) +{ + char *url; + errno_t rc; + + if (kmsurl == NULL) { + return NULL; + } + + url = kms->url; + switch (type) { + case SEL_MK: + rc = sprintf_s(url, KMS_URL_BUF_SZ, "%s/describe-key", kmsurl); + break; + case ENC_KEY: + rc = sprintf_s(url, KMS_URL_BUF_SZ, "%s/encrypt-datakey", kmsurl); + break; + case DEC_KEY: + rc = sprintf_s(url, KMS_URL_BUF_SZ, "%s/decrypt-datakey", kmsurl); + break; + default: + break; + } + km_securec_check_ss(rc, "", ""); + return url; +} + +static char *his_kms_mk_sel_reqbody(HisKmsMgr *kms, const char *keyid) +{ + const char *temp; + + temp = JS_TO_STR(( + { + "data": { + "attributes": { + "key_id":"{key_id}" + } + } + } + )); + + ReplaceRule rules[] = { + {"{key_id}", keyid} + }; + + km_safe_free(kms->reqbody); + kms->reqbody = json_replace(temp, rules, ARR_LEN(rules)); + + return kms->reqbody; +} + +#define LEN_BUF_SZ 12 +static char *his_kms_mk_enc_reqbody(HisKmsMgr *kms, const char *keyid, KmUnStr plain) +{ + const char *temp; + KmStr hex; + char hexlen[LEN_BUF_SZ]; + errno_t rc; + + temp = JS_TO_STR(( + { + "data": { + "attributes": { + "key_id":"{key_id}", + "plain_text": "{plain_encode}", + "datakey_plain_length": "{plain_encode_len}" + } + } + } + )); + + hex = km_hex_encode(plain); + if (hex.val == NULL) { + km_err_msg(kms->err, "when access '%s', failed to encode cipher", kms->url); + return NULL; + } + + rc = sprintf_s(hexlen, LEN_BUF_SZ, "%lu", hex.len); + km_securec_check_ss(rc, "", ""); + + ReplaceRule rules[] = { + {"{key_id}", keyid}, + {"{plain_encode}", hex.val}, + {"{plain_encode_len}", hexlen} + }; + + km_safe_free(kms->reqbody); + kms->reqbody = json_replace(temp, rules, ARR_LEN(rules)); + km_safe_free(hex.val); + + return kms->reqbody; +} + +static char *his_kms_mk_dec_reqbody(HisKmsMgr *kms, const char *keyid, KmUnStr cipher) +{ + const char *temp; + char hexlen[LEN_BUF_SZ]; + errno_t rc; + + temp = JS_TO_STR(( + { + "data": { + "attributes": { + "key_id":"{key_id}", + "cipher_text": "{cipher_encode}", + "datakey_plain_length": "{cipher_encode_len}" + } + } + } + )); + + KmStr hex = km_hex_encode(cipher); + if (hex.val == NULL) { + km_err_msg(kms->err, "when access '%s', failed to encode cipher", kms->url); + return NULL; + } + + rc = sprintf_s(hexlen, LEN_BUF_SZ, "%lu", hex.len); + km_securec_check_ss(rc, "", ""); + + ReplaceRule rules[] = { + {"{key_id}", keyid}, + {"{cipher_encode}", hex.val}, + {"{cipher_encode_len}", hexlen}, + }; + + km_safe_free(kms->reqbody); + kms->reqbody = json_replace(temp, rules, ARR_LEN(rules)); + km_safe_free(hex.val); + + return kms->reqbody; +} + +/* + * do - handle kms server error + * out - return : 1 (no error), 0 (http 300+ error), -1 (token expired) + */ +static int his_kms_handle_err(HisKmsMgr *kms) +{ + HttpMgr *http; + char *resbody; + long state; + char *kmserr; + + http = kms->httpmgr; + + state = httpmgr_get_status(http); + /* if succeed, just return */ + if (httpmgr_get_status(http) < HTTP_MIN_UNEXCEPTED_CODE) { + return 1; + } + + resbody = httpmgr_get_res_body(http); + kmserr = json_find(resbody, "code"); + if (kmserr != NULL && strcmp(kmserr, "KMS.3001") == 0) { + km_safe_free(kmserr); + return -1; + } + km_safe_free(kmserr); + + km_err_msg(kms->err, "failed to access '%s', http status: %ld, error message: %s", kms->url, state, resbody); + return 0; +} + +char *his_kms_mk_select(HisKmsMgr *kms, const char *keypath) +{ + HttpMgr *http; + char *resbody; + char *keystate; + int ret; + + HisKmsKeyPath kpath = his_kms_parser_key_path(kms, keypath); + if (km_err_catch(kms->err)) { + return NULL; + } + + http = kms->httpmgr; +#ifdef ENABLE_KM_DEBUG + httpmgr_set_output(http, "./kms.out"); +#endif + httpmgr_set_req_line(http, his_kms_get_url(kms, kpath.kmsurl, SEL_MK), CURLOPT_HTTPPOST, kms->cacert); + httpmgr_set_req_header(http, "Content-Type:application/json"); + httpmgr_set_req_header(http, his_iam_get_token(kms->iam)); + httpmgr_set_req_body(http, his_kms_mk_sel_reqbody(kms, kpath.keyid)); + char remain[3] = {0, 0, 1}; + httpmgr_set_response(http, remain, NULL); + + httpmgr_receive(http); + km_safe_free(kpath.kmsurl); + km_safe_free(kpath.keyid); + /* handle client or net error */ + if (km_err_catch(kms->err)) { + return NULL; + } + /* handle server error */ + ret = his_kms_handle_err(kms); + if (ret == -1) { + /* + * -1 means: token expired. we nee refresh the token and try again. + * we can make sure this function only be called once + */ + his_iam_refresh_token(kms->iam); + return his_kms_mk_select(kms, keypath); + } else if (ret == 0) { + return NULL; + } + + resbody = httpmgr_get_res_body(http); + keystate = json_find(resbody, "key_state"); + if (keystate == NULL) { + km_err_msg(kms->err, "when access '%s', failed find 'key_state' from http response body", kms->url); + return NULL; + } + + return keystate; +} + +KmUnStr his_kms_mk_encrypt(HisKmsMgr *kms, const char *keypath, KmUnStr plain) +{ + HttpMgr *http; + char *resbody; + KmStr hex = {0}; + KmUnStr cipher = {0}; + int ret; + char *len; + + if (plain.val == NULL) { + return cipher; + } + + HisKmsKeyPath kpath = his_kms_parser_key_path(kms, keypath); + if (km_err_catch(kms->err)) { + return cipher; + } + + http = kms->httpmgr; +#ifdef ENABLE_KM_DEBUG + httpmgr_set_output(http, "./kms.out"); +#endif + httpmgr_set_req_line(http, his_kms_get_url(kms, kpath.kmsurl, ENC_KEY), CURLOPT_HTTPPOST, kms->cacert); + httpmgr_set_req_header(http, "Content-Type:application/json"); + httpmgr_set_req_header(http, his_iam_get_token(kms->iam)); + httpmgr_set_req_body(http, his_kms_mk_enc_reqbody(kms, kpath.keyid, plain)); + + char remain[3] = {0, 0, 1}; + httpmgr_set_response(http, remain, NULL); + + httpmgr_receive(http); + km_safe_free(kpath.kmsurl); + km_safe_free(kpath.keyid); + /* check client error */ + if (km_err_catch(kms->err)) { + return cipher; + } + + /* check server error */ + ret = his_kms_handle_err(kms); + if (ret == -1) { + return his_kms_mk_encrypt(kms, keypath, plain); + } else if (ret == 0) { + return cipher; + } + + resbody = httpmgr_get_res_body(http); + + hex.val = json_find(resbody, "cipher_text"); + if (hex.val == NULL) { + km_err_msg(kms->err, "failed to find 'cipher_text' filed from http response body of '%s'.", kms->url); + return cipher; + } + + len = json_find(resbody, "datakey_length"); + if (len == NULL) { + km_free(hex.val); + km_err_msg(kms->err, "failed to find 'datakey_length' filed from http response body of '%s'.", kms->url); + return cipher; + } + hex.len = atoi(len); + cipher = km_hex_decode(hex); + km_free(hex.val); + km_free(len); + if (cipher.val == NULL) { + km_err_msg(kms->err, "failed to decode hex cipher in http response body of '%s'.", kms->url); + } + + return cipher; +} + +KmUnStr his_kms_mk_decrypt(HisKmsMgr *kms, const char *keypath, KmUnStr cipher) +{ + HttpMgr *http; + char *resbody; + KmStr hex; + KmUnStr plain = {0}; + int ret; + char *len; + + if (cipher.val == NULL) { + return plain; + } + + HisKmsKeyPath kpath = his_kms_parser_key_path(kms, keypath); + if (km_err_catch(kms->err)) { + return plain; + } + + http = kms->httpmgr; +#ifdef ENABLE_KM_DEBUG + httpmgr_set_output(http, "./kms.out"); +#endif + httpmgr_set_req_line(http, his_kms_get_url(kms, kpath.kmsurl, DEC_KEY), CURLOPT_HTTPPOST, kms->cacert); + httpmgr_set_req_header(http, "Content-Type:application/json"); + httpmgr_set_req_header(http, his_iam_get_token(kms->iam)); + httpmgr_set_req_body(http, his_kms_mk_dec_reqbody(kms, kpath.keyid, cipher)); + + char remain[3] = {0, 0, 1}; + httpmgr_set_response(http, remain, NULL); + + httpmgr_receive(http); + km_safe_free(kpath.kmsurl); + km_safe_free(kpath.keyid); + /* check client error */ + if (km_err_catch(kms->err)) { + return plain; + } + + /* check server error */ + ret = his_kms_handle_err(kms); + if (ret == -1) { + return his_kms_mk_decrypt(kms, keypath, cipher); + } else if (ret == 0) { + return plain; + } + + resbody = httpmgr_get_res_body(http); + hex.val = json_find(resbody, "data_key"); + if (hex.val == NULL) { + km_err_msg(kms->err, "failed to find 'data_key' filed from http resonpse of '%s'.", kms->url); + return plain; + } + + len = json_find(resbody, "datakey_length"); + if (len == NULL) { + km_free(hex.val); + km_err_msg(kms->err, "failed to find 'datakey_length' filed from http response body of '%s'.", kms->url); + return plain; + } + hex.len = atoi(len); + + plain = km_hex_decode(hex); + km_free(hex.val); + km_free(len); + if (plain.val == NULL) { + km_err_msg(kms->err, "failed to decode hex plain in http response body of '%s'.", kms->url); + } + + return plain; +} + +KmUnStr his_kms_dk_create(HisKmsMgr *kms, const char *keypath, KmUnStr *cipher) +{ +#define HIS_DK_DEFAULT_LEN 16 + + KmUnStr plain = {0}; + KmUnStr _cipher = {0}; + + if (cipher == NULL) { + return plain; + } + + plain.val = (unsigned char *)km_alloc(HIS_DK_DEFAULT_LEN); + if (plain.val == NULL) { + km_err_msg(kms->err, "failed to alloc memory."); + return plain; + } + + if (RAND_bytes(plain.val, HIS_DK_DEFAULT_LEN) != 1) { + km_err_msg(kms->err, "his kms failed to create data key."); + } + plain.len = HIS_DK_DEFAULT_LEN; + + _cipher = his_kms_mk_encrypt(kms, keypath, plain); + cipher->val = _cipher.val; + cipher->len = _cipher.len; + + return plain; +} \ No newline at end of file diff --git a/src/gausskernel/security/keymgr/hwc/security_hwc.cpp b/src/gausskernel/security/keymgr/hwc/security_hwc.cpp new file mode 100644 index 000000000..6c68c46b3 --- /dev/null +++ b/src/gausskernel/security/keymgr/hwc/security_hwc.cpp @@ -0,0 +1,241 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * security_hwc.cpp + * + * IDENTIFICATION + * src/gausskernel/security/keymgr/src/hwc/security_hwc.cpp + * + * ------------------------------------------------------------------------- + */ +#include "keymgr/hwc/security_hwc.h" +#include +#include +#include +#include "keymgr/security_key_mgr.h" +#include "keymgr/hwc/security_hwc_kms.h" +#include "keymgr/hwc/security_hwc_iam.h" + +HwcMgr *hwc_mgr_new(KmErr *err) +{ + HwcMgr *hwc; + + hwc = (HwcMgr *)km_alloc_zero(sizeof(HwcMgr)); + if (hwc == NULL) { + return NULL; + } + + hwc->kmgr.err = err; + hwc->iammgr = hwc_iam_new(err); + hwc->kmsmgr = hwc_kms_new(hwc->iammgr, err); + + if (hwc->iammgr == NULL || hwc->kmsmgr == NULL) { + hwc_mgr_free(hwc); + return NULL; + } + + return hwc; +} + +void hwc_mgr_read_env(HwcMgr *hwc) +{ + char *env; + KvScan *scan; + int kvcnt; + + if (hwc == NULL) { + return; + } + + if (hwc->setarg == 1) { + return; + } + + env = km_env_get("HUAWEI_KMS_INFO"); + if (env == NULL || strlen(env) == 0) { + km_err_msg(hwc->kmgr.err, + "please use environment variabl '$HUAWEI_KMS_INFO' to set identity and project information of kms."); + return; + } + + scan = kv_scan_init(env); + kvcnt = kv_scan_exec(scan); + for (;;) { + if (kvcnt > 0) { + hwc_mgr_set_arg(hwc, km_str_strip(scan->key), km_str_strip(scan->value)); + kvcnt = kv_scan_exec(scan); + } else if (kvcnt == 0) { + hwc_mgr_set_arg(hwc, km_str_strip(scan->key), km_str_strip(scan->value)); + break; + } else { + km_err_msg(hwc->kmgr.err, "parse key value error, error at position %d of '$HUAWEI_KMS_INFO'.", + scan->curpos); + break; + } + } + kv_scan_drop(scan); +} + +void hwc_mgr_free(HwcMgr *hwc) +{ + if (hwc == NULL) { + return; + } + + hwc_iam_free(hwc->iammgr); + hwc_kms_free(hwc->kmsmgr); + km_free(hwc); +} + +void hwc_mgr_set_arg(HwcMgr *hwc, const char *key, const char *value) +{ +#define MAX_ARG_LEN 512 + + if (hwc == NULL) { + return; + } + if (strlen(key) > MAX_ARG_LEN) { + return; + } + if (strlen(value) > MAX_ARG_LEN) { + return; + } + + hwc->setarg = 1; + hwc_iam_set_arg(hwc->iammgr, key, value); + hwc_kms_set_arg(hwc->kmsmgr, key, value); +} + +static KeyMgr *hwc_new(KmErr *err) +{ + return (KeyMgr *)hwc_mgr_new(err); +} + +static void hwc_free(KeyMgr *kmgr) +{ + hwc_mgr_free((HwcMgr *)(void *)kmgr); +} + +static void hwc_set_arg(KeyMgr *kmgr, const char *key, const char *value) +{ + hwc_mgr_set_arg((HwcMgr *)(void *)kmgr, key, value); +} + +static char *hwc_mk_select(KeyMgr *kmgr, KeyInfo key) +{ + HwcMgr *hwc = (HwcMgr *)(void *)kmgr; + + if (key.algo == NULL|| strlen(key.algo) == 0) { + km_err_msg(hwc->kmgr.err, "failed to access huawei_kms, please set key algorithm type."); + return NULL; + } + + if (key.id == NULL || strlen(key.id) == 0) { + km_err_msg(hwc->kmgr.err, "failed to access huawei_kms, please set key id."); + } + + if (strcasecmp(key.algo, "aes_256") != 0) { + km_err_msg(hwc->kmgr.err, "invalid algorithm '%s', huawei_kms only support 'aes_256'.", key.algo); + return NULL; + } + + hwc_mgr_read_env(hwc); + if (km_err_catch(hwc->kmgr.err)) { + return NULL; + } + + return hwc_kms_mk_select(hwc->kmsmgr, key.id); +} + +static KmUnStr hwc_mk_encrypt(KeyMgr *kmgr, KeyInfo key, KmUnStr plain) +{ +#define HIS_KMS_BLOCK_LEN 16 + HwcMgr *hwc = (HwcMgr *)(void *)kmgr; + KmUnStr cipher = {0}; + + if (plain.len % HIS_KMS_BLOCK_LEN != 0) { + km_err_msg(hwc->kmgr.err, "the length of plain we sent to huawei_kms should be exactly divided by 16."); + return cipher; + } + + hwc_mgr_read_env(hwc); + if (km_err_catch(hwc->kmgr.err)) { + return cipher; + } + + return hwc_kms_mk_encrypt(hwc->kmsmgr, key.id, plain); +} + +static KmUnStr hwc_mk_decrypt(KeyMgr *kmgr, KeyInfo key, KmUnStr cipher) +{ + HwcMgr *hwc = (HwcMgr *)(void *)kmgr; + KmUnStr plain = {0}; + + hwc_mgr_read_env(hwc); + if (km_err_catch(hwc->kmgr.err)) { + return plain; + } + + return hwc_kms_mk_decrypt(hwc->kmsmgr, key.id, cipher); +} + +static KmUnStr hwc_dk_create(KeyMgr *kmgr, KeyInfo key, KmUnStr *cipher) +{ + HwcMgr *hwc = (HwcMgr *)(void *)kmgr; + KmUnStr plain = {0}; + + hwc_mgr_read_env(hwc); + if (km_err_catch(hwc->kmgr.err)) { + return plain; + } + + return hwc_kms_dk_create(hwc->kmsmgr, key.id, cipher); +} + +KeyMethod huawei_kms = { + "huawei_kms", + + hwc_new, /* kmgr_new */ + hwc_free, /* kmgr_free */ + hwc_set_arg, /* kmgr_set_arg */ + + NULL, /* mk_create */ + NULL, /* mk_delete */ + hwc_mk_select, /* mk_select */ + hwc_mk_encrypt, /* mk_encrypt */ + hwc_mk_decrypt, /* mk_decrypt */ + + hwc_dk_create, /* dk_create */ +}; + +/* + * currently, most restful api of the Huawei Cloud Kms and Hybrid Cloud Kms are the same. + * we just reuse the methods here + */ +KeyMethod hcs_kms = { + "hcs_kms", + + hwc_new, /* kmgr_new */ + hwc_free, /* kmgr_free */ + hwc_set_arg, /* kmgr_set_arg */ + + NULL, /* mk_create */ + NULL, /* mk_delete */ + hwc_mk_select, /* mk_select */ + hwc_mk_encrypt, /* mk_encrypt */ + hwc_mk_decrypt, /* mk_decrypt */ + + hwc_dk_create, /* dk_create */ +}; \ No newline at end of file diff --git a/src/gausskernel/security/keymgr/hwc/security_hwc_iam.cpp b/src/gausskernel/security/keymgr/hwc/security_hwc_iam.cpp new file mode 100644 index 000000000..13d924bee --- /dev/null +++ b/src/gausskernel/security/keymgr/hwc/security_hwc_iam.cpp @@ -0,0 +1,271 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * security_hwc_iam.cpp + * + * IDENTIFICATION + * src/gausskernel/security/keymgr/src/hwc/security_hwc_iam.cpp + * + * ------------------------------------------------------------------------- + */ + +#include "keymgr/hwc/security_hwc_iam.h" +#include +#include +#include "keymgr/comm/security_http.h" +#include "keymgr/comm/security_utils.h" +#include "keymgr/comm/security_json.h" + +HwcIamMgr *hwc_iam_new(KmErr *errbuf) +{ + HwcIamMgr *iam; + + iam = (HwcIamMgr *)km_alloc_zero(sizeof(HwcIamMgr)); + if (iam == NULL) { + return NULL; + } + + iam->err = errbuf; + + iam->httpmgr = httpmgr_new(errbuf); + if (iam->httpmgr == NULL) { + km_free(iam); + return NULL; + } + + return iam; +} + +void hwc_iam_free(HwcIamMgr *iam) +{ + if (iam == NULL) { + return; + } + + km_safe_free(iam->username); + km_safe_free(iam->passwd); + km_safe_free(iam->domain); + km_safe_free(iam->project); + + km_safe_free(iam->url); + + httpmgr_free(iam->httpmgr); + km_safe_free(iam->cacert); + + km_safe_free(iam->reqbody); + km_safe_free(iam->token); + + km_free(iam); +} + +/* + * set iam authentication info, and cache them. + * when the token expired, we will use cached info to refresh token. + */ +void hwc_iam_set_arg(HwcIamMgr *iam, const char *key, const char *value) +{ + if (key == NULL || value == NULL) { + return; + } + + /* + * if we have cached the http reqbody and token. + * when we change any parameters, we need to clear the cache at the same time. + */ + km_safe_free(iam->reqbody); + km_safe_free(iam->token); + + if (strcasecmp(key, "iamuser") == 0) { + km_safe_free(iam->username); /* free the old one if need */ + iam->username = km_strdup(value); + } else if (strcasecmp(key, "iampassword") == 0) { + km_safe_free(iam->passwd); + iam->passwd = km_strdup(value); + } else if (strcasecmp(key, "iamdomain") == 0) { + km_safe_free(iam->domain); + iam->domain = km_strdup(value); + } else if (strcasecmp(key, "iamurl") == 0) { + km_safe_free(iam->url); + iam->url = km_strdup(value); + } else if (strcasecmp(key, "kmsproject") == 0) { + km_safe_free(iam->project); + iam->project = km_strdup(value); + } else if (strcasecmp(key, "iamcacert") == 0) { + km_safe_free(iam->cacert); + iam->cacert = km_realpath(value, iam->err); + } /* ignore unknowned para */ +} + +static void hwc_iam_has_set_auth_info(HwcIamMgr *iam) +{ + if (iam->username == NULL) { + km_err_msg(iam->err, "failed to access huawei cloud iam service, plase set parameter 'iamUser'"); + return; + } + if (iam->passwd == NULL) { + km_err_msg(iam->err, "failed to access huawei cloud iam service, plase set parameter 'iamPassword'"); + return; + } + if (iam->domain == NULL) { + km_err_msg(iam->err, "failed to access huawei cloud iam service, plase set parameter 'iamDomain'"); + return; + } + if (iam->project == NULL) { + km_err_msg(iam->err, "failed to access huawei cloud iam service, plase set parameter 'kmsProject'"); + return; + } +} + +static char *hwc_iam_get_url(HwcIamMgr *iam) +{ +#define HWC_IAM_URL_LEN 256 + if (iam->url == NULL) { + km_err_msg(iam->err, "failed to access huawei cloud iam service, plase set parameter 'iamUrl'"); + return NULL; + } + + if (strlen(iam->url) > HWC_IAM_URL_LEN) { + km_err_msg(iam->err, "failed to access huawei cloud iam service, the length of 'iamUrl' exceeds 256"); + return NULL; + } + + return iam->url; +} + +static char *hwc_iam_get_req_body(HwcIamMgr *iam) +{ + hwc_iam_has_set_auth_info(iam); + if (km_err_catch(iam->err)) { + return NULL; + } + + const char *reqtemp = JS_TO_STR(( + { + "auth": { + "identity": { + "methods": ["password"], + "password": { + "user": { + "name": "{username}", + "password": "{passwd}", + "domain": { + "name": "{domain}" + } + } + } + }, + "scope": { + "project": { + "name": "{projname}" + } + } + } + } + )); + + ReplaceRule rules[] = { + {"{username}", iam->username}, + {"{passwd}", iam->passwd}, + {"{projname}", iam->project}, + {"{domain}", iam->domain} + }; + km_safe_free(iam->reqbody); + iam->reqbody = json_replace(reqtemp, rules, ARR_LEN(rules)); + if (iam->reqbody == NULL) { + km_err_msg(iam->err, "when create iam request body, failed to parse json template"); + } + + return iam->reqbody; +} + +static char *hwc_iam_parse_token(HwcIamMgr *iam) +{ +#define MAX_TOKEN_LEN 8192 + char *resbody; + char *token; + long state; + errno_t rc; + size_t tkhdrlen; + + resbody = httpmgr_get_res_body(iam->httpmgr); + state = httpmgr_get_status(iam->httpmgr); + if (state >= HTTP_MIN_UNEXCEPTED_CODE) { + km_err_msg(iam->err, "failed to access '%s', http status: %ld, error message: %s", iam->url, state, resbody); + return NULL; + } + + token = httpmgr_get_res_header(iam->httpmgr); + if (token == NULL) { + km_err_msg(iam->err, "when access '%s', failed find 'access_token' from http response header", iam->url); + return NULL; + } + + /* we cache token until it gets expired */ + tkhdrlen = strlen("X-Auth-Token:%s ") + strlen(token) + 1; + if (tkhdrlen > MAX_TOKEN_LEN) { + km_err_msg(iam->err, "invalid token header len: %lu", tkhdrlen); + return NULL; + } + + iam->token = (char *)km_alloc(tkhdrlen); + if (iam->token == NULL) { + km_err_msg(iam->err, "malloc memory error"); + return NULL; + } + + rc = sprintf_s(iam->token, tkhdrlen, "X-Auth-Token:%s ", token + strlen("X-Subject-Token: ")); + km_securec_check_ss(rc, "", ""); + + return iam->token; +} + +char *hwc_iam_get_token(HwcIamMgr *iam) +{ + HttpMgr *http; + char *reqbody; + + /* if we have cache token, just return it. */ + if (iam->token != NULL) { + return iam->token; + } + + http = iam->httpmgr; +#ifdef ENABLE_KM_DEBUG + httpmgr_set_output(http, "./iam.out"); +#endif + httpmgr_set_req_line(http, hwc_iam_get_url(iam), CURLOPT_HTTPPOST, iam->cacert); + httpmgr_set_req_header(http, "Content-Type:application/json"); + reqbody = hwc_iam_get_req_body(iam); + httpmgr_set_req_header(http, httpmgr_get_req_body_len(http, reqbody)); + httpmgr_set_req_body(http, reqbody); + char remain[3] = {0, 1, 1}; + httpmgr_set_response(http, remain, "X-Subject-Token"); + + httpmgr_receive(http); + /* check client error here */ + if (km_err_catch(iam->err)) { + return NULL; + } + + /* check server error here, such as http 400, http 500 */ + return hwc_iam_parse_token(iam); +} + +char *hwc_iam_refresh_token(HwcIamMgr *iam) +{ + km_safe_free(iam->token); + + return hwc_iam_get_token(iam); +} \ No newline at end of file diff --git a/src/gausskernel/security/keymgr/hwc/security_hwc_kms.cpp b/src/gausskernel/security/keymgr/hwc/security_hwc_kms.cpp new file mode 100644 index 000000000..099bc19f3 --- /dev/null +++ b/src/gausskernel/security/keymgr/hwc/security_hwc_kms.cpp @@ -0,0 +1,625 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * security_hwc_kms.cpp + * + * IDENTIFICATION + * src/gausskernel/security/keymgr/src/hwc/security_hwc_kms.cpp + * + * ------------------------------------------------------------------------- + */ + +#include "keymgr/hwc/security_hwc_kms.h" +#include +#include "keymgr/comm/security_http.h" +#include "keymgr/comm/security_utils.h" +#include "keymgr/comm/security_json.h" +#include "keymgr/comm/security_encode.h" +#include "keymgr/hwc/security_hwc_iam.h" + +typedef struct { + char *kmsurl; + char *keyid; +} HwcKmsKeyPath; + +HwcKmsMgr *hwc_kms_new(HwcIamMgr *iam, KmErr *errbuf) +{ + HwcKmsMgr *kms; + + if (iam == NULL) { + return NULL; + } + + kms = (HwcKmsMgr *)km_alloc_zero(sizeof(HwcKmsMgr)); + if (kms == NULL) { + return NULL; + } + kms->err = errbuf; + kms->aksk = false; + kms->iam = iam; + kms->httpmgr = httpmgr_new(errbuf); + if (kms->httpmgr == NULL) { + km_free(kms); + return NULL; + } + + return kms; +} + +void hwc_kms_free(HwcKmsMgr *kms) +{ + if (kms == NULL) { + return; + } + km_safe_free(kms->reqbody); + + httpmgr_free(kms->httpmgr); + km_safe_free(kms->cacert); + + km_free(kms); +} + +void hwc_kms_set_arg(HwcKmsMgr *kms, const char *key, const char *value) +{ +#define HWC_PROJ_ID_MAX_LEN 96 + errno_t rc; + + if (key == NULL || value == NULL) { + return; + } + + if (strcasecmp(key, "kmscacert") == 0) { + km_safe_free(kms->cacert); /* free the old one if need */ + kms->cacert = km_realpath(value, kms->err); + } else if (strcasecmp(key, "kmsprojectid") == 0) { + if (strlen(value) > HWC_PROJ_ID_MAX_LEN) { + km_err_msg(kms->err, "the length of 'kmsProjectId' is too long, the maximum value is 96."); + return; + } + rc = sprintf_s(kms->projid, HWC_PROJ_ID_HDR_SZ, "x-project-id: %s", value); + km_securec_check_ss(rc, "", ""); + } else if (strcasecmp(key, "ak") == 0) { + httpmgr_ak_set(kms->httpmgr, value); + kms->aksk = true; + } else if (strcasecmp(key, "sk") == 0) { + httpmgr_sk_set(kms->httpmgr, value); + kms->aksk = true; + } /* ignore unknowned para */ +} + +/* the format of keypath is 'URL/ID', we need return 'KEYID' */ +static HwcKmsKeyPath hwc_kms_parser_key_path(HwcKmsMgr *kms, const char *keypath) +{ +#define HWC_KMS_URL_LEN 1024 + size_t split = 0; + HwcKmsKeyPath kpath = {0}; + size_t i; + + if (keypath == NULL || strlen(keypath) <= 1) { + km_err_msg(kms->err, "huawei kms failed to get keypath filed."); + return kpath; + } + + if (strlen(keypath) > HWC_KMS_URL_LEN) { + km_err_msg(kms->err, "failed to access huawei kms, the length of keypath exceeds 1024."); + return kpath; + } + + for (i = strlen(keypath) - 1; i > 0; i--) { + if (keypath[i] == '/') { + split = i; + break; + } + } + + if (split == 0 || split == strlen(keypath) - 1) { + km_err_msg(kms->err, "the format of keypath should be like 'KmsUrl/KeyId', but not '%s'.", keypath); + return kpath; + } + + kpath.keyid = km_strdup(keypath + split + 1); + kpath.kmsurl = km_strndup(keypath, split); + + return kpath; +} + +typedef enum { + SEL_MK = 0, + ENC_KEY, + DEC_KEY, + CRT_KEY +} UrlType; + +static char *hwc_kms_get_url(HwcKmsMgr *kms, const char *kmsurl, UrlType type) +{ + char *url; + errno_t rc; + + if (kmsurl == NULL) { + return NULL; + } + + url = kms->url; + switch (type) { + case SEL_MK: + rc = sprintf_s(url, HWC_KMS_URL_SZ, "%s/describe-key", kmsurl); + break; + case ENC_KEY: + rc = sprintf_s(url, HWC_KMS_URL_SZ, "%s/encrypt-datakey", kmsurl); + break; + case DEC_KEY: + rc = sprintf_s(url, HWC_KMS_URL_SZ, "%s/decrypt-datakey", kmsurl); + break; + case CRT_KEY: + rc = sprintf_s(url, HWC_KMS_URL_SZ, "%s/create-datakey", kmsurl); + break; + default: + break; + } + km_securec_check_ss(rc, "", ""); + + if (kms->aksk && strlen(kms->projid) == 0) { + km_err_msg(kms->err, "can't access '%s', please set arg 'kmsProjectId'.", url); + } + return url; +} + +static char *hwc_kms_mk_sel_reqbody(HwcKmsMgr *kms, const char *keyid) +{ + const char *temp; + + temp = JS_TO_STR(( + { + "key_id": "{key_id}" + } + )); + + ReplaceRule rules[] = { + {"{key_id}", keyid} + }; + + km_safe_free(kms->reqbody); + kms->reqbody = json_replace(temp, rules, ARR_LEN(rules)); + + return kms->reqbody; +} + +#define LEN_BUF_SZ 12 +static char *hwc_kms_mk_enc_reqbody(HwcKmsMgr *kms, const char *keyid, KmUnStr plain) +{ + const char *temp; + KmStr sha; + char shalen[LEN_BUF_SZ]; + errno_t rc; + + temp = JS_TO_STR(( + { + "key_id": "{key_id}", + "plain_text":"{plain_encode}", + "datakey_plain_length": "{plain_encode_len}" + } + )); + + sha = km_sha_encode(plain); + if (sha.val == NULL) { + km_err_msg(kms->err, "when access '%s', failed to encode cipher", kms->url); + return NULL; + } + + rc = sprintf_s(shalen, LEN_BUF_SZ, "%lu", plain.len); + km_securec_check_ss(rc, "", ""); + + ReplaceRule rules[] = { + {"{key_id}", keyid}, + {"{plain_encode}", sha.val}, + {"{plain_encode_len}", shalen} + }; + + km_safe_free(kms->reqbody); + kms->reqbody = json_replace(temp, rules, ARR_LEN(rules)); + km_free(sha.val); + + return kms->reqbody; +} + +static char *hwc_kms_mk_dec_reqbody(HwcKmsMgr *kms, const char *keyid, KmUnStr cipher) +{ +#define PLAIN_LEN 92 + const char *temp; + char plainlen[LEN_BUF_SZ]; + errno_t rc; + + temp = JS_TO_STR(( + { + "key_id": "{key_id}", + "cipher_text":"{cipher_encode}", + "datakey_cipher_length": "{cipher_encode_len}" + } + )); + + KmStr sha = km_sha_encode(cipher); + if (sha.val == NULL) { + km_err_msg(kms->err, "when access '%s', failed to encode cipher", kms->url); + return NULL; + } + + if (cipher.len <= PLAIN_LEN) { + km_err_msg(kms->err, "when access '%s', the length of cipher is too short: '%lu'.", kms->url, cipher.len); + return NULL; + } + + rc = sprintf_s(plainlen, LEN_BUF_SZ, "%lu", cipher.len - PLAIN_LEN); + km_securec_check_ss(rc, "", ""); + + ReplaceRule rules[] = { + {"{key_id}", keyid}, + {"{cipher_encode}", sha.val}, + {"{cipher_encode_len}", plainlen}, + }; + + km_safe_free(kms->reqbody); + kms->reqbody = json_replace(temp, rules, ARR_LEN(rules)); + km_free(sha.val); + + return kms->reqbody; +} + +static char *hwc_kms_dk_crt_reqbody(HwcKmsMgr *kms, const char *keyid) +{ + const char *temp; + + temp = JS_TO_STR(( + { + "datakey_length": "128", + "key_id": "{key_id}" + } + )); + + ReplaceRule rules[] = { + {"{key_id}", keyid} + }; + + km_safe_free(kms->reqbody); + kms->reqbody = json_replace(temp, rules, ARR_LEN(rules)); + return kms->reqbody; +} + +/* + * do - handle kms server error + * out - return : 1 (no error), 0 (http 300+ error), -1 (token expired) + */ +static int hwc_kms_handle_err(HwcKmsMgr *kms) +{ + HttpMgr *http; + char *resbody; + long state; + char *kmserr; + + http = kms->httpmgr; + + state = httpmgr_get_status(http); + /* if succeed, just return */ + if (httpmgr_get_status(http) < HTTP_MIN_UNEXCEPTED_CODE) { + return 1; + } + + resbody = httpmgr_get_res_body(http); + kmserr = json_find(resbody, "error_code"); + if (kmserr != NULL && strcmp(kmserr, "KMS.0303") == 0) { + km_safe_free(kmserr); + return -1; + } + km_safe_free(kmserr); + + km_err_msg(kms->err, "failed to access '%s', http status: %ld, error message: %s", kms->url, state, resbody); + return 0; +} + +static char *hwc_kms_key_state(char *keystate) +{ + if (keystate == NULL) { + return NULL; + } + + /* + * key state: + * 1: to be active + * 2: active + * 3: disable + * 4: to be deleted + * 5: to be improved + * do not use atoi() here, we get the 'keystate' from http message, the 'keystate' could be anything + */ + if (strcmp(keystate, "1") == 0) { + return km_strdup("to be active"); + } else if (strcmp(keystate, "2") == 0) { + return km_strdup("active"); + } else if (strcmp(keystate, "3") == 0) { + return km_strdup("disable"); + } else if (strcmp(keystate, "4") == 0) { + return km_strdup("to be active"); + } else if (strcmp(keystate, "5") == 0) { + return km_strdup("to be improved"); + } else { + return km_strdup(keystate); + } +} + +char *hwc_kms_mk_select(HwcKmsMgr *kms, const char *keypath) +{ + HttpMgr *http; + char *reqbody; + char *resbody; + char *keystate; + char *state; + int ret; + + HwcKmsKeyPath kpath = hwc_kms_parser_key_path(kms, keypath); + if (km_err_catch(kms->err)) { + return NULL; + } + + http = kms->httpmgr; +#ifdef ENABLE_KM_DEBUG + httpmgr_set_output(http, "./kms.out"); +#endif + httpmgr_set_req_line(http, hwc_kms_get_url(kms, kpath.kmsurl, SEL_MK), CURLOPT_HTTPPOST, kms->cacert); + httpmgr_set_req_header(http, "Content-Type:application/json"); + kms->aksk ? httpmgr_set_req_header(http, kms->projid) : httpmgr_set_req_header(http, hwc_iam_get_token(kms->iam)); + reqbody = hwc_kms_mk_sel_reqbody(kms, kpath.keyid); + httpmgr_set_req_header(http, httpmgr_get_req_body_len(http, reqbody)); + httpmgr_set_req_body(http, reqbody); + char remain[3] = {0, 0, 1}; + httpmgr_set_response(http, remain, NULL); + + httpmgr_receive(http); + km_safe_free(kpath.kmsurl); + km_safe_free(kpath.keyid); + /* handle client or net error */ + if (km_err_catch(kms->err)) { + return NULL; + } + /* handle server error */ + ret = hwc_kms_handle_err(kms); + if (ret == -1) { + /* + * -1 means: token expired. we nee refresh the token and try again. + * we can make sure thwc function only be called once + */ + hwc_iam_refresh_token(kms->iam); + return hwc_kms_mk_select(kms, keypath); + } else if (ret == 0) { + return NULL; + } + + resbody = httpmgr_get_res_body(http); + keystate = json_find(resbody, "key_state"); + if (keystate == NULL) { + km_err_msg(kms->err, "when access '%s', failed find 'key_state' from http response body", kms->url); + return NULL; + } + + state = hwc_kms_key_state(keystate); + km_free(keystate); + + return state; +} + +KmUnStr hwc_kms_mk_encrypt(HwcKmsMgr *kms, const char *keypath, KmUnStr plain) +{ + HttpMgr *http; + char *reqbody; + char *resbody; + KmStr sha = {0}; + KmUnStr cipher = {0}; + int ret; + + if (plain.val == NULL) { + return cipher; + } + + HwcKmsKeyPath kpath = hwc_kms_parser_key_path(kms, keypath); + if (km_err_catch(kms->err)) { + return cipher; + } + + http = kms->httpmgr; +#ifdef ENABLE_KM_DEBUG + httpmgr_set_output(http, "./kms.out"); +#endif + httpmgr_set_req_line(http, hwc_kms_get_url(kms, kpath.kmsurl, ENC_KEY), CURLOPT_HTTPPOST, kms->cacert); + reqbody = hwc_kms_mk_enc_reqbody(kms, kpath.keyid, plain); + httpmgr_set_req_header(http, httpmgr_get_req_body_len(http, reqbody)); + httpmgr_set_req_header(http, "Content-Type:application/json"); + kms->aksk ? httpmgr_set_req_header(http, kms->projid) : httpmgr_set_req_header(http, hwc_iam_get_token(kms->iam)); + httpmgr_set_req_body(http, reqbody); + + char remain[3] = {0, 0, 1}; + httpmgr_set_response(http, remain, NULL); + + httpmgr_receive(http); + km_safe_free(kpath.kmsurl); + km_safe_free(kpath.keyid); + /* check client error */ + if (km_err_catch(kms->err)) { + return cipher; + } + + /* check server error */ + ret = hwc_kms_handle_err(kms); + if (ret == -1) { + return hwc_kms_mk_encrypt(kms, keypath, plain); + } else if (ret == 0) { + return cipher; + } + + resbody = httpmgr_get_res_body(http); + sha.val = json_find(resbody, "cipher_text"); + if (sha.val == NULL) { + km_err_msg(kms->err, "failed to find 'cipher_text' filed from http response body of '%s'.", kms->url); + return cipher; + } + sha.len = strlen(sha.val); + cipher = km_sha_decode(sha); + km_free(sha.val); + if (cipher.val == NULL) { + km_err_msg(kms->err, "failed to decode hex cipher in http response body of '%s'.", kms->url); + } + + return cipher; +} + +KmUnStr hwc_kms_mk_decrypt(HwcKmsMgr *kms, const char *keypath, KmUnStr cipher) +{ + HttpMgr *http; + char *reqbody; + char *resbody; + KmStr sha; + KmUnStr plain = {0}; + int ret; + + if (cipher.val == NULL) { + return plain; + } + + HwcKmsKeyPath kpath = hwc_kms_parser_key_path(kms, keypath); + if (km_err_catch(kms->err)) { + return plain; + } + + http = kms->httpmgr; +#ifdef ENABLE_KM_DEBUG + httpmgr_set_output(http, "./kms.out"); +#endif + httpmgr_set_req_line(http, hwc_kms_get_url(kms, kpath.kmsurl, DEC_KEY), CURLOPT_HTTPPOST, kms->cacert); + reqbody = hwc_kms_mk_dec_reqbody(kms, kpath.keyid, cipher); + httpmgr_set_req_header(http, httpmgr_get_req_body_len(http, reqbody)); + httpmgr_set_req_header(http, "Content-Type:application/json"); + kms->aksk ? httpmgr_set_req_header(http, kms->projid) : httpmgr_set_req_header(http, hwc_iam_get_token(kms->iam)); + httpmgr_set_req_body(http, reqbody); + + char remain[3] = {0, 0, 1}; + httpmgr_set_response(http, remain, NULL); + + httpmgr_receive(http); + km_safe_free(kpath.kmsurl); + km_safe_free(kpath.keyid); + /* check client error */ + if (km_err_catch(kms->err)) { + return plain; + } + + /* check server error */ + ret = hwc_kms_handle_err(kms); + if (ret == -1) { + return hwc_kms_mk_decrypt(kms, keypath, cipher); + } else if (ret == 0) { + return plain; + } + + resbody = httpmgr_get_res_body(http); + sha.val = json_find(resbody, "data_key"); + if (sha.val == NULL) { + km_err_msg(kms->err, "failed to find 'data_key' filed from http resonpse of '%s'.", kms->url); + return plain; + } + sha.len = strlen(sha.val); + + plain = km_hex_decode(sha); + km_free(sha.val); + if (plain.val == NULL) { + km_err_msg(kms->err, "failed to decode hex plain in http response body of '%s'.", kms->url); + } + + return plain; +} + +KmUnStr hwc_kms_dk_create(HwcKmsMgr *kms, const char *keypath, KmUnStr *cipher) +{ + HttpMgr *http; + char *reqbody; + char *resbody; + KmUnStr plain = {0}; + char *hexplain; + KmStr sha; + KmUnStr _cipher; + int ret; + + HwcKmsKeyPath kpath = hwc_kms_parser_key_path(kms, keypath); + if (km_err_catch(kms->err)) { + return plain; + } + + http = kms->httpmgr; +#ifdef ENABLE_KM_DEBUG + httpmgr_set_output(http, "./kms.out"); +#endif + httpmgr_set_req_line(http, hwc_kms_get_url(kms, kpath.kmsurl, CRT_KEY), CURLOPT_HTTPPOST, kms->cacert); + reqbody = hwc_kms_dk_crt_reqbody(kms, kpath.keyid); + httpmgr_set_req_header(http, httpmgr_get_req_body_len(http, reqbody)); + httpmgr_set_req_header(http, "Content-Type:application/json"); + if (!kms->aksk) { + httpmgr_set_req_header(http, hwc_iam_get_token(kms->iam)); + } + httpmgr_set_req_body(http, reqbody); + + char remain[3] = {0, 0, 1}; + httpmgr_set_response(http, remain, NULL); + + httpmgr_receive(http); + km_safe_free(kpath.kmsurl); + km_safe_free(kpath.keyid); + /* check client error */ + if (km_err_catch(kms->err)) { + return plain; + } + + /* check server error */ + ret = hwc_kms_handle_err(kms); + if (ret == -1) { + return hwc_kms_dk_create(kms, keypath, cipher); + } else if (ret == 0) { + return plain; + } + + resbody = httpmgr_get_res_body(http); + hexplain = json_find(resbody, "plain_text"); + if (hexplain == NULL) { + km_err_msg(kms->err, "failed to find 'plain_text' filed from http response body of '%s'.", kms->url); + } + + sha.val = json_find(resbody, "cipher_text"); + if (sha.val == NULL) { + km_safe_free(hexplain); + km_err_msg(kms->err, "failed to find 'cipher_text' filed from http response body of '%s'.", kms->url); + return plain; + } + sha.len = strlen(sha.val); + + plain = km_hex_decode({hexplain, strlen(hexplain)}); + _cipher = km_sha_decode(sha); + km_free(hexplain); + km_free(sha.val); + if (plain.val == NULL || _cipher.val == NULL) { + km_safe_free(plain.val); + km_safe_free(_cipher.val); + km_err_msg(kms->err, "failed to decode hex data key plain and cipher in http response body of '%s'.", kms->url); + } + + cipher->val = _cipher.val; + cipher->len = _cipher.len; + + return plain; +} \ No newline at end of file diff --git a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/register_gs_ktool.cpp b/src/gausskernel/security/keymgr/ktool/security_gs_ktool.cpp similarity index 51% rename from src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/register_gs_ktool.cpp rename to src/gausskernel/security/keymgr/ktool/security_gs_ktool.cpp index 6f34e3122..82ad6482d 100644 --- a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/register_gs_ktool.cpp +++ b/src/gausskernel/security/keymgr/ktool/security_gs_ktool.cpp @@ -13,7 +13,7 @@ * See the Mulan PSL v2 for more details. * ------------------------------------------------------------------------- * - * register_gs_ktool.cpp + * security_gs_ktool.cpp * gs_ktool is an independent key management tool provided by GaussDB Kernel, can generate and store symmetric * key with [16, 112] bytes. * when CREATE CMKO, if KEY_STROE = gs_ktool, then: @@ -22,43 +22,39 @@ * available for AES_256 algorithm * if you register gs_ktool, you should be sure your system has installed gs_ktool, and the environment variables * and the configuration files are available. - * + * * IDENTIFICATION - * src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/register_gs_ktool.cpp + * src/gausskernel/security/keymgr/src/ktool/security_gs_ktool.cpp * * ------------------------------------------------------------------------- */ - -#include "cmkem_version_control.h" -#ifdef ENABLE_GS_KTOOL - -#include "register_gs_ktool.h" +#include "keymgr/ktool/security_gs_ktool.h" #include -#include #include +#ifdef ENABLE_KT #include "gs_ktool/kt_interface.h" -#include "reg_hook_frame.h" -#include "cmkem_comm_algorithm.h" +#endif +#include "keymgr/security_key_mgr.h" +#include "keymgr/comm/security_error.h" +#include "keymgr/localkms/security_cmkem_comm_algorithm.h" const int MAX_KEYPATH_LEN = 64; const int UPPER_TO_LOWER_OFFSET = 'a' - 'A'; -static CmkCacheList *cmk_cache_list = NULL; -static const char *supported_algorithms[] = {"AES_256_CBC", "SM4", NULL}; +static const char *g_support_algo[] = {"AES_256_CBC", "SM4", "AES_256_GCM", NULL}; -static CmkemErrCode check_cmk_id_validity(CmkIdentity *cmk_identity); +static CmkemErrCode check_cmk_id_validity(KeyInfo info); static CmkemErrCode check_cmk_algo_validity(const char *cmk_algo); -static CmkemErrCode check_cmk_entity_validity(CmkIdentity *cmk_identity); +static CmkemErrCode check_cmk_entity_validity(KeyInfo info); static void get_cmk_id_from_key_path(const char *key_path, unsigned int *cmk_id); -static CmkemErrCode read_cmk_plain(unsigned int cmk_id, CmkemUStr **cmk_plain); +static CmkemErrCode read_cmk_plain(GsKtoolMgr *ktool, unsigned int cmk_id, CmkemUStr **cmk_plain); -static ProcessPolicy create_cmk_obj_hookfunc(CmkIdentity *cmk_identity); -static ProcessPolicy encrypt_cek_plain_hookfunc(CmkemUStr *cek_plain, CmkIdentity *cmk_identity, - CmkemUStr **cek_cipher); -static ProcessPolicy decrypt_cek_cipher_hookfunc(CmkemUStr *cek_cipher, CmkIdentity *cmk_identity, - CmkemUStr **cek_plain); +CmkCacheList *init_cmk_cache_list(); +void push_cmk_to_cache(GsKtoolMgr *ktool, unsigned int cmk_id, const unsigned char *cmk_plian); +bool get_cmk_from_cache(GsKtoolMgr *ktool, unsigned int cmk_id, unsigned char *cmk_plain); +void free_cmk_cache_list(CmkCacheList *cmk_cahce_list); -static void cmkem_tolower(const char *in, char *out, size_t out_buf_len) +void cmkem_tolower(const char *in, char *out, size_t out_buf_len) { if (out_buf_len < strlen(in) + 1) { return; @@ -75,21 +71,20 @@ static void cmkem_tolower(const char *in, char *out, size_t out_buf_len) out[i] = '\0'; } -static CmkemErrCode check_cmk_id_validity(CmkIdentity *cmk_identity) +static CmkemErrCode check_cmk_id_validity(KeyInfo info) { const char *key_path_tag = "gs_ktool/"; char tmp_str[MAX_KEYPATH_LEN] = {0}; int tmp_pos = 0; bool has_invalid_char = false; - const char *cmk_id_str = cmk_identity->cmk_id_str; - if (strlen(cmk_id_str) <= strlen(key_path_tag)) { - cmkem_errmsg("invalid key path: '%s', it should be like \"%s1\".", cmk_identity->cmk_id_str, key_path_tag); + if (strlen(info.id) <= strlen(key_path_tag)) { + cmkem_errmsg("invalid key path: '%s', it should be like \"%s1\".", info.id, key_path_tag); return CMKEM_CHECK_CMK_ID_ERR; } - char cmk_id_lower[strlen(cmk_id_str) + 1] = {0}; - cmkem_tolower(cmk_id_str, cmk_id_lower, sizeof(cmk_id_lower)); + char cmk_id_lower[strlen(info.id) + 1] = {0}; + cmkem_tolower(info.id, cmk_id_lower, sizeof(cmk_id_lower)); for (size_t i = 0; i < strlen(key_path_tag); i++) { if (cmk_id_lower[i] != key_path_tag[i]) { cmkem_errmsg("invalid key path: '%s', it should be like \"%s1\".", cmk_id_lower, key_path_tag); @@ -97,75 +92,84 @@ static CmkemErrCode check_cmk_id_validity(CmkIdentity *cmk_identity) } } - for (size_t i = strlen(key_path_tag); i < strlen(cmk_id_str); i++) { - if (cmk_id_str[i] < '0' || cmk_id_str[i] > '9') { + for (size_t i = strlen(key_path_tag); i < strlen(info.id); i++) { + if (info.id[i] < '0' || info.id[i] > '9') { has_invalid_char = true; } - tmp_str[tmp_pos] = cmk_id_str[i]; + tmp_str[tmp_pos] = info.id[i]; tmp_pos++; } if (has_invalid_char) { - cmkem_errmsg("invalid key path: '%s', '%s' is expected to be an integer.", cmk_identity->cmk_id_str, tmp_str); + cmkem_errmsg("invalid key path: '%s', '%s' is expected to be an integer.", info.id, tmp_str); return CMKEM_CHECK_CMK_ID_ERR; } tmp_str[tmp_pos] = '\0'; - cmk_identity->cmk_id_num = atoi(tmp_str); return CMKEM_SUCCEED; } +unsigned int get_key_id(const char *str) +{ + if (strlen(str) <= strlen("gs_ktool/")) { + return 0; + } + return (unsigned int)atoi(str + strlen("gs_ktool/")); +} + static CmkemErrCode check_cmk_algo_validity(const char *cmk_algo) { char error_msg_buf[MAX_CMKEM_ERRMSG_BUF_SIZE] = {0}; error_t rc = 0; - for (size_t i = 0; supported_algorithms[i] != NULL; i++) { - if (strcasecmp(cmk_algo, supported_algorithms[i]) == 0) { + for (size_t i = 0; g_support_algo[i] != NULL; i++) { + if (strcasecmp(cmk_algo, g_support_algo[i]) == 0) { return CMKEM_SUCCEED; } } rc = sprintf_s(error_msg_buf, MAX_CMKEM_ERRMSG_BUF_SIZE, "unpported algorithm '%s', gs_ktool only support: ", cmk_algo); - securec_check_ss_c(rc, "", ""); + km_securec_check_ss(rc, "", ""); - for (size_t i = 0; supported_algorithms[i] != NULL; i++) { - rc = strcat_s(error_msg_buf, MAX_CMKEM_ERRMSG_BUF_SIZE, supported_algorithms[i]); - securec_check_c(rc, "\0", "\0"); + for (size_t i = 0; g_support_algo[i] != NULL; i++) { + rc = strcat_s(error_msg_buf, MAX_CMKEM_ERRMSG_BUF_SIZE, g_support_algo[i]); + km_securec_check(rc, "", ""); rc = strcat_s(error_msg_buf, MAX_CMKEM_ERRMSG_BUF_SIZE, " "); - securec_check_c(rc, "\0", "\0"); + km_securec_check(rc, "", ""); } - cmkem_errmsg(error_msg_buf); + cmkem_errmsg("%s", error_msg_buf); return CMKEM_CHECK_ALGO_ERR; } -static CmkemErrCode check_cmk_entity_validity(CmkIdentity *cmk_identity) +static CmkemErrCode check_cmk_entity_validity(KeyInfo info) { +#ifdef ENABLE_KT unsigned int cmk_len = 0; - - if (!get_cmk_len(cmk_identity->cmk_id_num, &cmk_len)) { - cmkem_errmsg("failed to read cmk from gs_ktool, key id: %d.", cmk_identity->cmk_id_num); + unsigned int cmk_id = get_key_id(info.id); + if (!get_cmk_len(cmk_id, &cmk_len)) { + cmkem_errmsg("failed to read cmk from gs_ktool, key id: %u.", cmk_id); return CMKEM_GS_KTOOL_ERR; } - if (cmk_len != get_key_len_by_algo(get_algo_by_str(cmk_identity->cmk_algo))) { + if (cmk_len < get_key_len_by_algo(get_algo_by_str(info.algo))) { return CMKEM_CHECK_ALGO_ERR; } - +#endif return CMKEM_SUCCEED; } static void get_cmk_id_from_key_path(const char *key_path, unsigned int *cmk_id) { const char *key_path_tag = "gs_ktool/"; - *cmk_id = (unsigned int) atoi(key_path + strlen(key_path_tag)); + *cmk_id = (unsigned int)atoi(key_path + strlen(key_path_tag)); } -static CmkemErrCode read_cmk_plain(unsigned int cmk_id, CmkemUStr **cmk_plain) +static CmkemErrCode read_cmk_plain(GsKtoolMgr *ktool, unsigned int cmk_id, CmkemUStr **cmk_plain) { +#ifdef ENABLE_KT unsigned int tmp_cmk_len = 0; *cmk_plain = malloc_cmkem_ustr(AES256_KEY_BUF_LEN); @@ -174,17 +178,18 @@ static CmkemErrCode read_cmk_plain(unsigned int cmk_id, CmkemUStr **cmk_plain) } /* case a : try to get cmk plain from cache */ - if (!get_cmk_from_cache(cmk_cache_list, cmk_id, (*cmk_plain)->ustr_val)) { + if (!get_cmk_from_cache(ktool, cmk_id, (*cmk_plain)->ustr_val)) { /* case b : failed to get cmk plian from cache, try to get it from gs_ktool */ if (!get_cmk_plain(cmk_id, (*cmk_plain)->ustr_val, &tmp_cmk_len)) { free_cmkem_ustr_with_erase(*cmk_plain); return CMKEM_GS_KTOOL_ERR; } - push_cmk_to_cache(cmk_cache_list, cmk_id, (*cmk_plain)->ustr_val); + push_cmk_to_cache(ktool, cmk_id, (*cmk_plain)->ustr_val); } (*cmk_plain)->ustr_len = (size_t) tmp_cmk_len; +#endif return CMKEM_SUCCEED; } @@ -192,7 +197,7 @@ static CmkemErrCode read_cmk_plain(unsigned int cmk_id, CmkemUStr **cmk_plain) CmkCacheList *init_cmk_cache_list() { CmkCacheList *cmk_cache_list = NULL; - cmk_cache_list = (CmkCacheList *)malloc(sizeof(CmkCacheList)); + cmk_cache_list = (CmkCacheList *)km_alloc(sizeof(CmkCacheList)); if (cmk_cache_list == NULL) { cmkem_errmsg("failed to malloc memory."); return NULL; @@ -203,12 +208,13 @@ CmkCacheList *init_cmk_cache_list() return cmk_cache_list; } -void push_cmk_to_cache(CmkCacheList *cmk_cache_list, unsigned int cmk_id, const unsigned char *cmk_plian) +void push_cmk_to_cache(GsKtoolMgr *ktool, unsigned int cmk_id, const unsigned char *cmk_plian) { + CmkCacheList *cache_list = ktool->cache; CmkCacheNode *new_node = NULL; CmkCacheNode *last_node = NULL; - new_node = (CmkCacheNode *)malloc(sizeof(CmkCacheNode)); + new_node = (CmkCacheNode *)km_alloc(sizeof(CmkCacheNode)); if (new_node == NULL) { cmkem_errmsg("failed to malloc memory."); return; @@ -218,31 +224,36 @@ void push_cmk_to_cache(CmkCacheList *cmk_cache_list, unsigned int cmk_id, const new_node->cmk_plain[i] = cmk_plian[i]; } - if (cmk_cache_list->cmk_node_cnt < MAX_CMK_CACHE_NODE_CNT) { - new_node->next = cmk_cache_list->first_cmk_node; - cmk_cache_list->first_cmk_node = new_node; - cmk_cache_list->cmk_node_cnt++; + if (cache_list->cmk_node_cnt < MAX_CMK_CACHE_NODE_CNT) { + new_node->next = cache_list->first_cmk_node; + cache_list->first_cmk_node = new_node; + cache_list->cmk_node_cnt++; } else { - last_node = cmk_cache_list->first_cmk_node; + last_node = cache_list->first_cmk_node; while (last_node->next->next != NULL) { last_node = last_node->next; } cmkem_free(last_node->next); last_node->next = NULL; + + new_node->next = cache_list->first_cmk_node; + cache_list->first_cmk_node = new_node; + cache_list->cmk_node_cnt++; } } -bool get_cmk_from_cache(CmkCacheList *cmk_cache_list, unsigned int cmk_id, unsigned char *cmk_plain) +bool get_cmk_from_cache(GsKtoolMgr *ktool, unsigned int cmk_id, unsigned char *cmk_plain) { + CmkCacheList *cache_list = ktool->cache; CmkCacheNode *cur_cmk_node = NULL; CmkCacheNode *correct_cmk_node = NULL; - if (cmk_cache_list->first_cmk_node == NULL) { + if (cache_list->first_cmk_node == NULL) { return false; } /* the head node is not used */ - cur_cmk_node = cmk_cache_list->first_cmk_node; + cur_cmk_node = cache_list->first_cmk_node; /* a. there are only 1 node */ if (cur_cmk_node->next == NULL) { if (cur_cmk_node->cmk_id == cmk_id) { @@ -252,11 +263,11 @@ bool get_cmk_from_cache(CmkCacheList *cmk_cache_list, unsigned int cmk_id, unsig return true; } } else { /* case b : there are 2 or more nodes */ - /* - * if the first node is the correct node, like this : + /* + * if the first node is the correct node, like this : * to find node '2', and the cache list is : '2' -> '1' -> '3' */ - if (cur_cmk_node->cmk_id == cmk_id) { + if (cur_cmk_node->cmk_id == cmk_id) { for (size_t i = 0; i < DEFAULT_CMK_CACHE_LEN; i++) { cmk_plain[i] = cur_cmk_node->cmk_plain[i]; } @@ -271,8 +282,8 @@ bool get_cmk_from_cache(CmkCacheList *cmk_cache_list, unsigned int cmk_id, unsig /* refresh cache list */ cur_cmk_node->next = correct_cmk_node->next; - correct_cmk_node->next = cmk_cache_list->first_cmk_node; - cmk_cache_list->first_cmk_node = correct_cmk_node; + correct_cmk_node->next = cache_list->first_cmk_node; + cache_list->first_cmk_node = correct_cmk_node; return true; } @@ -300,137 +311,159 @@ void free_cmk_cache_list(CmkCacheList *cmk_cahce_list) cmkem_free(to_free); } - cmkem_free(cmk_cahce_list); + km_free(cmk_cahce_list); } -static ProcessPolicy create_cmk_obj_hookfunc(CmkIdentity *cmk_identity) + +char *ktool_mk_select(KeyMgr *kmgr, KeyInfo info) { + GsKtoolMgr *kt = (GsKtoolMgr *)(void *)kmgr; CmkemErrCode ret = CMKEM_SUCCEED; - if (cmk_identity->cmk_store == NULL || strcasecmp(cmk_identity->cmk_store, "gs_ktool") != 0) { - return POLICY_CONTINUE; - } - - if (!init_gs_ktool()) { - return POLICY_ERROR; - } - - if (cmk_identity->cmk_id_str == NULL) { - cmkem_errmsg("failed to create client master key, failed to find arg: KEY_PATH."); - return POLICY_ERROR; + if (info.id == NULL) { + km_err_msg(kt->kmgr.err, "failed to create client master key, failed to find arg: KEY_PATH."); + return NULL; } - if (cmk_identity->cmk_algo == NULL) { - cmkem_errmsg("failed to create client master key, failed to find arg: ALGORITHM."); - return POLICY_ERROR; + if (info.algo == NULL) { + km_err_msg(kt->kmgr.err, "failed to create client master key, failed to find arg: ALGORITHM."); + return NULL; } - ret = check_cmk_algo_validity(cmk_identity->cmk_algo); + ret = check_cmk_algo_validity(info.algo); if (ret != CMKEM_SUCCEED) { - return POLICY_ERROR; + km_err_msg(kt->kmgr.err, "%s", get_cmkem_errmsg(ret)); + return NULL; } - ret = check_cmk_id_validity(cmk_identity); + ret = check_cmk_id_validity(info); if (ret != CMKEM_SUCCEED) { - return POLICY_ERROR; + km_err_msg(kt->kmgr.err, "%s", get_cmkem_errmsg(ret)); + return NULL; } - ret = check_cmk_entity_validity(cmk_identity); + ret = check_cmk_entity_validity(info); if (ret != CMKEM_SUCCEED) { - return POLICY_ERROR; + km_err_msg(kt->kmgr.err, "%s", get_cmkem_errmsg(ret)); + return NULL; } - return POLICY_BREAK; + return km_strdup("active"); } -static ProcessPolicy encrypt_cek_plain_hookfunc(CmkemUStr *cek_plain, CmkIdentity *cmk_identity, CmkemUStr **cek_cipher) +KmUnStr ktool_mk_encrypt(KeyMgr *kmgr, KeyInfo info, KmUnStr plain) { + GsKtoolMgr *kt = (GsKtoolMgr *)(void *)kmgr; CmkemErrCode ret = CMKEM_SUCCEED; + KmUnStr cipher = {0}; + CmkemUStr _plain = {plain.val, plain.len}; + CmkemUStr *_cipher = NULL; + CmkemUStr *cmk_plain = NULL; unsigned int cmk_id = 0; - AlgoType cmk_algo = get_algo_by_str(cmk_identity->cmk_algo); - - if (cmk_identity->cmk_store == NULL || strcasecmp(cmk_identity->cmk_store, "gs_ktool") != 0) { - return POLICY_CONTINUE; - } + AlgoType cmk_algo = get_algo_by_str(info.algo); - if (!init_gs_ktool()) { - return POLICY_ERROR; - } + get_cmk_id_from_key_path(info.id, &cmk_id); - get_cmk_id_from_key_path(cmk_identity->cmk_id_str, &cmk_id); - - ret = read_cmk_plain(cmk_id, &cmk_plain); + ret = read_cmk_plain(kt, cmk_id, &cmk_plain); if (ret != CMKEM_SUCCEED) { - return POLICY_ERROR; + km_err_msg(kt->kmgr.err, "%s", get_cmkem_errmsg(ret)); + return cipher; } - ret = encrypt_with_symm_algo(cmk_algo, cek_plain, cmk_plain, cek_cipher); + ret = encrypt_with_symm_algo(cmk_algo, &_plain, cmk_plain, &_cipher); free_cmkem_ustr_with_erase(cmk_plain); - if (ret != CMKEM_SUCCEED) { - return POLICY_ERROR; + if (ret != CMKEM_SUCCEED) { + km_err_msg(kt->kmgr.err, "%s", get_cmkem_errmsg(ret)); + return cipher; } - return POLICY_BREAK; + cipher.val = _cipher->ustr_val; + cipher.len = _cipher->ustr_len; + km_free(_cipher); + return cipher; } -static ProcessPolicy decrypt_cek_cipher_hookfunc(CmkemUStr *cek_cipher, CmkIdentity *cmk_identity, - CmkemUStr **cek_plain) +KmUnStr ktool_mk_decrypt(KeyMgr *kmgr, KeyInfo info, KmUnStr cipher) { + GsKtoolMgr *kt = (GsKtoolMgr *)(void *)kmgr; CmkemErrCode ret = CMKEM_SUCCEED; + KmUnStr plain = {0}; + CmkemUStr _cipher = {cipher.val, cipher.len}; + CmkemUStr *_plain = NULL; + CmkemUStr *cmk_plain = NULL; unsigned int cmk_id = 0; - AlgoType cmk_algo = get_algo_by_str(cmk_identity->cmk_algo); + AlgoType cmk_algo = get_algo_by_str(info.algo); - if (cmk_identity->cmk_store == NULL || strcasecmp(cmk_identity->cmk_store, "gs_ktool") != 0) { - return POLICY_CONTINUE; - } - - if (!init_gs_ktool()) { - return POLICY_ERROR; - } + get_cmk_id_from_key_path(info.id, &cmk_id); - get_cmk_id_from_key_path(cmk_identity->cmk_id_str, &cmk_id); - - ret = read_cmk_plain(cmk_id, &cmk_plain); + ret = read_cmk_plain(kt, cmk_id, &cmk_plain); if (ret != CMKEM_SUCCEED) { - return POLICY_ERROR; + km_err_msg(kt->kmgr.err, "%s", get_cmkem_errmsg(ret)); + return plain; } - ret = decrypt_with_symm_algo(cmk_algo, cek_cipher, cmk_plain, cek_plain); + ret = decrypt_with_symm_algo(cmk_algo, &_cipher, cmk_plain, &_plain); free_cmkem_ustr_with_erase(cmk_plain); - if (ret != CMKEM_SUCCEED) { - return POLICY_ERROR; + if (ret != CMKEM_SUCCEED) { + km_err_msg(kt->kmgr.err, "%s", get_cmkem_errmsg(ret)); + return plain; } - - return POLICY_BREAK; + + plain.val = _plain->ustr_val; + plain.len = _plain->ustr_len; + km_free(_plain); + return plain; } -int reg_cmke_manager_gs_ktool_main() +KeyMgr* ktool_new(KmErr *err) { - /* init LRU cache list */ - cmk_cache_list = init_cmk_cache_list(); - if (cmk_cache_list == NULL) { - return -1; +#ifdef ENABLE_KT + if (!init_gs_ktool()) { + km_err_msg(err, "failed to init gs_ktool."); + return NULL; } - - CmkEntityManager gs_ktool = { - create_cmk_obj_hookfunc, - encrypt_cek_plain_hookfunc, - decrypt_cek_cipher_hookfunc, - NULL, /* drop_cmk_obj_hook_func: no need */ - NULL, /* post_create_cmk_obj_hook_func: no need */ - }; - - return (reg_cmk_entity_manager(gs_ktool) == CMKEM_SUCCEED) ? 0 : -1; +#endif + GsKtoolMgr *ktmgr; + + ktmgr = (GsKtoolMgr *)km_alloc_zero(sizeof(GsKtoolMgr)); + if (ktmgr == NULL) { + km_err_msg(err, "failed to malloc memory"); + } + + ktmgr->kmgr.err = err; + ktmgr->cache = init_cmk_cache_list(); + + return (KeyMgr *)ktmgr; } -void exit_cmke_manager_gs_ktool() +static void ktool_free(KeyMgr *ktmgr) { - free_cmk_cache_list(cmk_cache_list); - cmk_cache_list = NULL; - + if (ktmgr == NULL) { + return; + } + + GsKtoolMgr *kt = (GsKtoolMgr *)(void *)ktmgr; + free_cmk_cache_list(kt->cache); +#ifdef ENABLE_KT deinit_gs_ktool(); +#endif + km_free(kt); } -#endif /* ENABLE_GS_KTOOL */ +KeyMethod gs_ktool = { + "gs_ktool", + + ktool_new, /* kmgr_new */ + ktool_free, /* kmgr_free */ + NULL, /* kmgr_set_arg */ + + NULL, /* mk_create */ + NULL, /* mk_delete */ + ktool_mk_select, /* mk_select */ + ktool_mk_encrypt, /* mk_encrypt */ + ktool_mk_decrypt, /* mk_decrypt */ + + NULL, /* dk_create */ +}; diff --git a/src/gausskernel/security/keymgr/localkms/security_cmkem_comm.cpp b/src/gausskernel/security/keymgr/localkms/security_cmkem_comm.cpp new file mode 100644 index 000000000..2c8b72aee --- /dev/null +++ b/src/gausskernel/security/keymgr/localkms/security_cmkem_comm.cpp @@ -0,0 +1,197 @@ +/* + * Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * security_cmkem_comm.cpp + * some common functions, include: + * 1. error code and error process + * 2. string process + * 3. format and conversion + * + * IDENTIFICATION + * src/gausskernel/security/keymgr/src/localkms/security_cmkem_comm.cpp + * + * ------------------------------------------------------------------------- + */ + +#include "keymgr/localkms/security_cmkem_comm.h" + +#include +#include +#include +#include +#include +#include +#include +#include "securec.h" +#include "securec_check.h" +#include "keymgr/comm/security_error.h" + +static char cmkem_errmsg_buf[MAX_CMKEM_ERRMSG_BUF_SIZE] = {0}; + +/* + * in some cases, where there is an error, we will report the error in detail. + * if not, we will use simple and general error information according to the error code + */ +static CmkemErrMsg simple_errmsg_list[] = { + {CMKEM_SUCCEED, "no error message, the result is 'SUCCEED'."}, + {CMKEM_UNKNOWN_ERR, "unknown error."}, + {CMKEM_CHECK_STORE_ERR, "failed to check the value of arg: 'KEY_STORE'."}, + {CMKEM_CHECK_ALGO_ERR, "failed to check the value of arg: 'ALGORITHM'."}, + {CMKEM_CHECK_CMK_ID_ERR, "failed to check the value of arg: 'KEY_PATH'."}, + {CMKEM_CHECK_IDENTITY_ERR, "faiel to check the usability of cmk entity."}, + {CMKEM_CHECK_INPUT_AUTH_ERR, "the authentication parameters are illegal."}, + {CMKEM_CHECK_PROJECT_NAME_ERR, "failed to check HuaWei KMS project name."}, + {CMKEM_GET_TOKEN_ERR, "failed to get token."}, + {CMKEM_WRITE_TO_BIO_ERR, "failed to wirite key to BIO buffer."}, + {CMKEM_READ_FROM_BIO_ERR, "failed to read key from BIO buffer."}, + {CMKEM_GET_RAND_NUM_ERR, "failed to generate a random num."}, + {CMKEM_DERIVED_KEY_ERR, "failed to derived key."}, + {CMKEM_GEN_RSA_KEY_ERR, "failed to create rsa key pair."}, + {CMKEM_GEN_SM2_KEY_ERR, "failed to create sm2 key pair."}, + {CMKEM_RSA_DECRYPT_ERR, "failed to decrypt cipher with RSA algorithm."}, + {CMKEM_RSA_ENCRYPT_ERR, "failed to encrypt plain with RSA algorithm."}, + {CMKEM_CHECK_HASH_ERR, "failed check the hash of data."}, + {CMKEM_EVP_ERR, "openssl evp error."}, + {CMKEM_SM2_ENC_ERR, "failed to encrypt plain with SM2 algorithm."}, + {CMKEM_SM2_DEC_ERR, "failed to decrypt cipher with SM2 algorithm."}, + {CMKEM_REG_CMK_MANAGER_ERR, "failed to register cmk entity manager."}, + {CMKEM_REGISTRE_FUNC_ERR, "failed to register hook function of cmk manager."}, + {CMKEM_CREATE_CMK_ERR, "failed to create client master key object."}, + {CMKEM_ENCRYPT_CEK_ERR, "failed to encrypt column encryption key plain with client master key."}, + {CMKEM_DECRYPT_CEK_ERR, "failed to decrypt column encryption key cipher with client master key."}, + {CMKEM_DROP_CMK_ERR, "failed to drop client master key object."}, + {CMKEM_GET_ENV_VAL_ERR, "failed to get environment value from system."}, + {CMKEM_CHECK_ENV_VAL_ERR, "the environment value is illegal."}, + {CMKEM_SET_ENV_VALUE_ERR, "failed to set global environment value."}, + {CMKEM_GS_KTOOL_ERR, "gs_ktool error."}, + {CMKEM_IAM_SERVER_ERR, "Huawei IAM server error."}, + {CMKEM_CHECK_CHACHE_ID_ERR, "the client cache id is invalid."}, + {CMKEM_CACHE_IS_EMPTY, "the specific cache block is empty."}, + {CMKEM_TOKEN_EXPIRED_ERR, "the token has expired."}, + {CMKEM_KMS_SERVER_ERR, "Huawei KMS server error."}, + {CMKEM_ENC_CMK_ERR, "failed to encrypt cmk entity plain."}, + {CMKEM_DEC_CMK_ERR, "failed to decrypt cmk entity cipher."}, + {CMKEM_CREATE_FILE_ERR, "failed to create file."}, + {CMKEM_FIND_FILE_ERR, "failed to find file."}, + {CMKEM_OPEN_FILE_ERR, "failed to open file."}, + {CMKEM_READ_FILE_ERR, "failed to read file."}, + {CMKEM_READ_FILE_STATUS_ERR, "failed to read file status."}, + {CMKEM_WRITE_FILE_ERR, "failed to write to file."}, + {CMKEM_REMOVE_FILE_ERR, "failed to remove file."}, + {CMKEM_MALLOC_MEM_ERR, "failed to malloc memory from heap."}, + {CMKEM_CHECK_BUF_LEN_ERR, "the buffer size is too short."}, + {CMKEM_CJSON_PARSE_ERR, "failed to parser string with 'CJSON'."}, + {CMKEM_FIND_CSJON_ERR, "failed to find node from cjson tree."}, + {CMKEM_SET_CJSON_VALUE_ERR, "failed to set the value of cjson tree node."}, + {CMKEM_CURL_INIT_ERR, "failed to init curl."}, + {CMKEM_CURL_ERR, "curl error."}, +}; + +/* + * write error message to err_msg buffer, if need error message, can chose: + * 1. cmkem_list_print error message. + * 2. copy/dump error message to new err_msg buffer. + */ +void write_cmkem_errmsg(const char *errmsg) +{ + errno_t rc = 0; + + if (cmkem_errmsg_buf == NULL) { + return; + } + + rc = memset_s(cmkem_errmsg_buf, MAX_CMKEM_ERRMSG_BUF_SIZE, 0, sizeof(cmkem_errmsg_buf)); + km_securec_check(rc, "", ""); + + rc = sprintf_s(cmkem_errmsg_buf, MAX_CMKEM_ERRMSG_BUF_SIZE, "%s", errmsg); + km_securec_check_ss(rc, "", ""); +} + +const char *get_cmkem_errmsg(CmkemErrCode err_code) +{ + if (cmkem_errmsg_buf == NULL) { + return NULL; + } + + if (strlen(cmkem_errmsg_buf) == 0) { + return simple_errmsg_list[err_code].err_msg; + } + + return (const char *)cmkem_errmsg_buf; +} + +void erase_data(void *data, size_t data_len) +{ + errno_t rc = 0; + + if (data == NULL) { + return; + } + + rc = memset_s(data, data_len, 0, data_len); + km_securec_check(rc, "", ""); +} + +void free_cmkem_str(CmkemStr *advstr_ptr) +{ + if (advstr_ptr != NULL) { + km_safe_free(advstr_ptr->str_val); + km_safe_free(advstr_ptr); + } + + advstr_ptr = NULL; +} + +CmkemUStr *malloc_cmkem_ustr(size_t ust_buf_len) +{ + if (ust_buf_len == 0) { + return NULL; + } + + CmkemUStr *key_str = (CmkemUStr *)km_alloc_zero(sizeof(CmkemUStr)); + if (key_str == NULL) { + return NULL; + } + + key_str->ustr_val = (unsigned char *)km_alloc_zero(ust_buf_len); + if (key_str->ustr_val == NULL) { + km_safe_free(key_str); + return NULL; + } + + key_str->ustr_len = 0; + return key_str; +} + +void free_cmkem_ustr(CmkemUStr *cmkem_ustr) +{ + if (cmkem_ustr != NULL) { + km_safe_free(cmkem_ustr->ustr_val); + km_safe_free(cmkem_ustr); + } + + cmkem_ustr = NULL; +} + +void free_cmkem_ustr_with_erase(CmkemUStr *cmkem_ustr) +{ + if (cmkem_ustr != NULL) { + erase_data(cmkem_ustr->ustr_val, cmkem_ustr->ustr_len); + km_safe_free(cmkem_ustr->ustr_val); + km_safe_free(cmkem_ustr); + } + + cmkem_ustr = NULL; +} diff --git a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/cmkem_comm_algorithm.cpp b/src/gausskernel/security/keymgr/localkms/security_cmkem_comm_algorithm.cpp similarity index 66% rename from src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/cmkem_comm_algorithm.cpp rename to src/gausskernel/security/keymgr/localkms/security_cmkem_comm_algorithm.cpp index 33ead3fe4..ff9be0df5 100644 --- a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/cmkem_comm_algorithm.cpp +++ b/src/gausskernel/security/keymgr/localkms/security_cmkem_comm_algorithm.cpp @@ -13,39 +13,41 @@ * See the Mulan PSL v2 for more details. * ------------------------------------------------------------------------- * - * cmkem_comm_algorithm.cpp + * security_cmkem_comm_algorithm.cpp * some general encryption and decryption function. * you can use them to encrypt and decrypt your data, CEK entity and CMK entity. - * + * * IDENTIFICATION - * src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/cmkem_comm_algorithm.cpp + * src/gausskernel/security/keymgr/src/localkms/security_cmkem_comm_algorithm.cpp * * ------------------------------------------------------------------------- */ -#include "cmkem_comm_algorithm.h" +#include "keymgr/localkms/security_cmkem_comm_algorithm.h" #include #include #include #include #include -#include "encrypt_decrypt.h" -#include "aead_aes_hamc_enc_key.h" +#include "keymgr/encrypt/security_encrypt_decrypt.h" +#include "keymgr/encrypt/security_aead_aes_hamc_enc_key.h" size_t get_key_len_by_algo(AlgoType cmk_algo) { switch (cmk_algo) { - case AES_256_CBC: - return 32; - case SM4: + case AT_AES_256_CBC: return 32; - case RSA_2048: + case AT_SM4_CBC: + return 16; + case AT_RSA_2048: return 256; - case RSA_3072: + case AT_RSA_3072: return 384; - case SM2: + case AT_SM2: return 32; + case AT_AES_256_GCM: + return 32; /* gcm security key length is 32 * 8 = 256 */ default: break; } @@ -56,70 +58,42 @@ size_t get_key_len_by_algo(AlgoType cmk_algo) AlgoType get_algo_by_str(const char *algo_str) { if (strcasecmp(algo_str, "AES_256_CBC") == 0) { - return AES_256_CBC; + return AT_AES_256_CBC; } else if (strcasecmp(algo_str, "SM4") == 0) { - return SM4; + return AT_SM4_CBC; } else if (strcasecmp(algo_str, "RSA_2048") == 0) { - return RSA_2048; + return AT_RSA_2048; } else if (strcasecmp(algo_str, "RSA_3072") == 0) { - return RSA_3072; + return AT_RSA_3072; } else if (strcasecmp(algo_str, "SM2") == 0) { - return SM2; - } - + return AT_SM2; + } else if (strcasecmp(algo_str, "AES_256_GCM") == 0) { + return AT_AES_256_GCM; + } + return UNKNOWN_ALGO; } ColumnEncryptionAlgorithm get_algo_combination(AlgoType algo) { switch (algo) { - case AES_256_CBC: + case AT_AES_256_CBC: return AEAD_AES_256_CBC_HMAC_SHA256; - case SM4: + case AT_SM4_CBC: return SM4_SM3; + case AT_AES_256_GCM: + return AES_256_GCM_ALGO; + case AT_AES_256_CTR: + return AES_256_CTR_ALGO; default: return INVALID_ALGORITHM; } } -CmkemErrCode get_hex_join_hash_from_ustr(CmkemUStr *ustr, CmkemStr **hex_join_hash) -{ - CmkemUStr *ustr_join_hash = NULL; - CmkemErrCode ret = CMKEM_SUCCEED; - - ret = join_sha256(ustr, &ustr_join_hash); - check_cmkem_ret(ret); - - *hex_join_hash = ustr_to_hex(ustr_join_hash); - free_cmkem_ustr_with_erase(ustr_join_hash); - if (*hex_join_hash == NULL) { - return CMKEM_MALLOC_MEM_ERR; - } - - return CMKEM_SUCCEED; -} - -CmkemErrCode get_ustr_from_hex_join_hash(CmkemStr *hex_join_hash, CmkemUStr **ustr) -{ - CmkemUStr *ustr_join_hash = NULL; - CmkemErrCode ret = CMKEM_SUCCEED; - - ustr_join_hash = hex_to_ustr(hex_join_hash); - if (ustr_join_hash == NULL) { - return CMKEM_MALLOC_MEM_ERR; - } - - ret = strip_sha256(ustr_join_hash, ustr); - free_cmkem_ustr_with_erase(ustr_join_hash); - check_cmkem_ret(ret); - - return CMKEM_SUCCEED; -} - CmkemErrCode encrypt_with_symm_algo(AlgoType algo, CmkemUStr *plain, CmkemUStr *key, CmkemUStr **cipher) { int cipher_len = 0; - AeadAesHamcEncKey derived_key = AeadAesHamcEncKey(key->ustr_val, get_key_len_by_algo(AES_256_CBC)); + AeadAesHamcEncKey derived_key = AeadAesHamcEncKey(key->ustr_val, get_key_len_by_algo(algo)); *cipher = malloc_cmkem_ustr(get_cipher_text_size(plain->ustr_len)); if (*cipher == NULL) { @@ -147,7 +121,7 @@ CmkemErrCode decrypt_with_symm_algo(AlgoType algo, CmkemUStr *cipher, CmkemUStr { int plain_len = 0; - AeadAesHamcEncKey derived_key = AeadAesHamcEncKey(key->ustr_val, get_key_len_by_algo(AES_256_CBC)); + AeadAesHamcEncKey derived_key = AeadAesHamcEncKey(key->ustr_val, get_key_len_by_algo(algo)); *plain = malloc_cmkem_ustr(cipher->ustr_len); /* the plain cmkem_list_len must be less than cipher cmkem_list_len */ if (*plain == NULL) { @@ -176,7 +150,12 @@ RSA *create_rsa_keypair(size_t rsa_key_len) RSA *rsa_key_pair = NULL; rsa_n_val = BN_new(); + if (rsa_n_val == NULL) { + return NULL; + } + if (BN_set_word(rsa_n_val, RSA_F4) != 1) { /* public exponent - 65537 */ + BN_free(rsa_n_val); return NULL; } @@ -197,7 +176,7 @@ RSA *create_rsa_keypair(size_t rsa_key_len) } CmkemErrCode write_rsa_keypair_to_biobuf(RSA *rsa_key_pair, BIO **pub_key_biobuf, BIO **priv_key_biobuf) -{ +{ int ssl_ret = 0; *pub_key_biobuf = BIO_new(BIO_s_mem()); @@ -245,42 +224,3 @@ CmkemErrCode read_rsa_key_from_biobuf(BIO *key_biobuf, CmkemUStr **key) return CMKEM_SUCCEED; } - -CmkemErrCode join_sha256(CmkemUStr *data, CmkemUStr **data_join_sha256) -{ - *data_join_sha256 = malloc_cmkem_ustr(data->ustr_len + SHA256_HASH_LEN + 1); - if (*data_join_sha256 == NULL) { - return CMKEM_MALLOC_MEM_ERR; - } - - for (size_t i = 0; i < data->ustr_len; i++) { - (*data_join_sha256)->ustr_val[i] = data->ustr_val[i]; - } - - SHA256(data->ustr_val, data->ustr_len, (*data_join_sha256)->ustr_val + data->ustr_len); - - (*data_join_sha256)->ustr_len = data->ustr_len + SHA256_HASH_LEN; - - return CMKEM_SUCCEED; -} - -CmkemErrCode strip_sha256(CmkemUStr *data_join_sha256, CmkemUStr **data) -{ - unsigned char expected_hash[SHA256_HASH_LEN + 1] = {0}; - size_t data_len = data_join_sha256->ustr_len - SHA256_HASH_LEN; - - SHA256(data_join_sha256->ustr_val, data_len, expected_hash); - - *data = malloc_cmkem_ustr(data_len + 1); - if (*data == NULL) { - return CMKEM_MALLOC_MEM_ERR; - } - - for (size_t i = 0; i < data_len; i++) { - (*data)->ustr_val[i] = data_join_sha256->ustr_val[i]; - } - (*data)->ustr_val[data_len] = '\0'; - (*data)->ustr_len = data_len; - - return CMKEM_SUCCEED; -} diff --git a/src/gausskernel/security/keymgr/localkms/security_file_enc.cpp b/src/gausskernel/security/keymgr/localkms/security_file_enc.cpp new file mode 100644 index 000000000..2978504ff --- /dev/null +++ b/src/gausskernel/security/keymgr/localkms/security_file_enc.cpp @@ -0,0 +1,409 @@ +/* + * Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * security_file_enc.cpp + * + * IDENTIFICATION + * src/gausskernel/security/keymgr/localkms/security_file_enc.cpp + * + * ------------------------------------------------------------------------- + */ + + +#include "keymgr/localkms/security_file_enc.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "securec.h" +#include "securec_check.h" +#include "keymgr/comm/security_utils.h" +#include "keymgr/encrypt/security_aead_aes_hamc_enc_key.h" +#include "keymgr/encrypt/security_encrypt_decrypt.h" + +const int KEY_METERIAL_LEN = 32; +const int DRIVED_KEY_LEN = 64; +const int ITERATE_ROUD = 10000; +const int KEY_FILE_HEADER_LEN = 20; + +static char g_env_value[PATH_MAX] = {0}; + +/* check the environment values input from the system */ +static CmkemErrCode check_env_value(const char *path_value) +{ + const char danger_char_list[] = {'|', ';', '&', '$', '<', '>', '`', '\\', '\'', '\"', '{', '}', '(', ')', '[', ']', + '~', '*', '?', '!'}; + + for (size_t i = 0; i < strlen(path_value); i++) { + for (size_t j = 0; j < sizeof(danger_char_list); j++) { + if (path_value[i] == danger_char_list[j]) { + cmkem_errmsg("the path '%s' contains invalid character '%c'.", path_value, path_value[i]); + return CMKEM_CHECK_ENV_VAL_ERR; + } + } + } + + return CMKEM_SUCCEED; +} + +/* + * keys generated by localkms are stored under the path ：$LOCALKMS_FILE_PATH/ + * in addition, if $LOCALKMS_FILE_PATH cannot be found, we will try to find $GAUSSHOME + */ +CmkemErrCode set_global_env_value() +{ + char *local_kms_path = NULL; + char tmp_gausshome_buf[PATH_MAX] = {0}; + errno_t rc = 0; + bool is_get_localkms_env = false; + char tmp_env_val[PATH_MAX] = {0}; + CmkemErrCode ret = CMKEM_SUCCEED; + + if (strlen(g_env_value) > 0) { + return CMKEM_SUCCEED; + } + + local_kms_path = km_env_get("LOCALKMS_FILE_PATH"); + if (local_kms_path == NULL || realpath(local_kms_path, tmp_env_val) == NULL) { + /* fail to get LOCALKMS_FILE_PATH, then try to get GAUSSHOME */ + local_kms_path = km_env_get("GAUSSHOME"); + if (local_kms_path != NULL) { + rc = sprintf_s(tmp_gausshome_buf, sizeof(tmp_gausshome_buf), "%s/%s", local_kms_path, "/etc/localkms"); + securec_check_ss_c(rc, "", ""); + + /* judge whether the $GAUSSHOME is obtained or not */ + if (realpath(tmp_gausshome_buf, tmp_env_val) != NULL) { + is_get_localkms_env = true; + } + } + } else { + is_get_localkms_env = true; + } + + if (!is_get_localkms_env) { + cmkem_errmsg("failed to get the environment value : '%s' or the path : '%s'.", "$LOCALKMS_FILE_PATH", + "$GAUSSHOME/etc/localkms/"); + return CMKEM_GET_ENV_VAL_ERR; + } + + ret = check_env_value(tmp_env_val); + check_cmkem_ret(ret); + + rc = strcpy_s(g_env_value, PATH_MAX, tmp_env_val); + securec_check_c(rc, "", ""); + + return CMKEM_SUCCEED; +} + +/* before creating or reading cmk and rand file, we will check if the files already exist */ +CmkemErrCode check_file_exist(const char *real_file_path, CheckKeyFileType chk_type) +{ + bool is_file_exist = false; + struct stat statbuf; + + is_file_exist = (lstat(real_file_path, &statbuf) < 0) ? false : true; + if (chk_type == CREATE_KEY_FILE && is_file_exist) { + cmkem_errmsg("cannot create file, the file '%s' already exists.\n", real_file_path); + return CMKEM_CREATE_FILE_ERR; + } else if (chk_type == READ_KEY_FILE && !is_file_exist) { + cmkem_errmsg("cannot read file, failed to find file '%s'.\n", real_file_path); + return CMKEM_FIND_FILE_ERR; + } + + return CMKEM_SUCCEED; +} + +/* + * if para:is_write_header == true, we will create a file header, and write the size of conent to header + * after that, if trying to read all content from file, we will read header to calcule the size of content + * buffer before reading all content. + * + * although it's redundant to add a header, in order to keep the compatibility with the old version, we want + * to keep using this way + */ +static CmkemErrCode create_file_and_write(const char *real_path, const unsigned char *content, size_t content_len, + bool is_write_header) +{ + int fd = 0; + char head[KEY_FILE_HEADER_LEN] = {0}; + errno_t rc = 0; + + fd = open(real_path, O_CREAT | O_RDWR, S_IRUSR | S_IWUSR); + if (fd == -1) { + cmkem_errmsg("failed to create file '%s'.\n", real_path); + return CMKEM_CREATE_FILE_ERR; + } + + if (is_write_header) { + rc = sprintf_s(head, sizeof(head), "%lu", content_len); + securec_check_ss_c(rc, "", ""); + write(fd, head, sizeof(head)); + } + + write(fd, content, content_len); + close(fd); + + return CMKEM_SUCCEED; +} + +/* + * in SQL : + * CREATE CLIENT MASTER KEY xxx (KEY_STORE = localkms, KEY_PATH = $key_path_value, ...) + * for each $key_path_value, we will create 4 file: + * 1. $LOCALKMS_FILE_PATH/$key_path_value.pub ： store public key cipher of asymmetric key + * 2. $LOCALKMS_FILE_PATH/$key_path_value.pub.rand ： store rands that are used to derive a key to encrypt + * public key plain + * 3. $LOCALKMS_FILE_PATH/$key_path_value.priv ： store private key cipher of asymmetric key + * 2. $LOCALKMS_FILE_PATH/$key_path_value.priv.rand ： store rands that are used to derive a key to encrypt + * private key plain + */ +void get_file_path_from_cmk_id(const char *cmk_id_str, LocalkmsFileType file_type, char *file_path_buf, size_t buf_len) +{ + error_t rc = 0; + const char *file_extension = NULL; + + switch (file_type) { + case PUB_KEY_FILE: + file_extension = ".pub"; + break; + case PRIV_KEY_FILE: + file_extension = ".priv"; + break; + case PUB_KEY_ENC_IV_FILE: + file_extension = ".pub.rand"; + break; + case PRIV_KEY_ENC_IV_FILE: + file_extension = ".priv.rand"; + break; + default: + break; + } + + rc = sprintf_s(file_path_buf, buf_len, "%s/%s%s", g_env_value, cmk_id_str, file_extension); + securec_check_ss_c(rc, "", ""); +} + +CmkemErrCode encrypt_and_write_key(const char *key_file_path, CmkemUStr *key_plain) +{ + unsigned char iv[KEY_METERIAL_LEN] = {0}; + unsigned char salt[KEY_METERIAL_LEN] = {0}; + unsigned char iv_salt_buf[sizeof(iv) + sizeof(salt)] = {0}; + unsigned char derived_key[DRIVED_KEY_LEN] = {0}; + unsigned char tmp_cipher[RSA2048_KEN_LEN] = {0}; + int tmp_cipher_len = 0; + char rand_file_path[PATH_MAX] = {0}; + errno_t rc = 0; + CmkemErrCode ret = CMKEM_SUCCEED; + + if (RAND_priv_bytes(iv, sizeof(iv)) != 1 || RAND_priv_bytes(salt, sizeof(salt)) != 1) { + return CMKEM_DERIVED_KEY_ERR; + } + + if (PKCS5_PBKDF2_HMAC((const char *)iv, sizeof(iv), salt, sizeof(salt), ITERATE_ROUD, EVP_sha256(), + sizeof(derived_key), derived_key) != 1) { + return CMKEM_DERIVED_KEY_ERR; + } + + AeadAesHamcEncKey derived_aead_key = AeadAesHamcEncKey(derived_key, DRIVED_KEY_LEN); + tmp_cipher_len = encrypt_data(key_plain->ustr_val, key_plain->ustr_len, derived_aead_key, + EncryptionType::DETERMINISTIC_TYPE, tmp_cipher, AEAD_AES_256_CBC_HMAC_SHA256); + if (tmp_cipher_len <= 0) { + return CMKEM_ENC_CMK_ERR; + } + + ret = create_file_and_write(key_file_path, tmp_cipher, (size_t)tmp_cipher_len, true); + if (ret != CMKEM_SUCCEED) { + return ret; + } + + rc = sprintf_s(rand_file_path, PATH_MAX, "%s.rand", key_file_path); + securec_check_ss_c(rc, "", ""); + + for (size_t i = 0; i < sizeof(iv); i++) { + iv_salt_buf[i] = iv[i]; + } + for (size_t i = 0; i < sizeof(salt); i++) { + iv_salt_buf[sizeof(iv) + i] = salt[i]; + } + + return create_file_and_write(rand_file_path, iv_salt_buf, sizeof(iv_salt_buf), true); +} + +/* the input file must contain a header which store the size value of file content */ +static CmkemErrCode read_content_from_file(const char *real_path, unsigned char *buf, const size_t buf_len, + size_t *content_len) +{ + int fd = 0; + char header[KEY_FILE_HEADER_LEN] = {0}; + + fd = open(real_path, O_RDONLY, 0); + if (fd < 0) { + cmkem_errmsg("failed to open file '%s'.\n", real_path); + return CMKEM_OPEN_FILE_ERR; + } + + if (read(fd, header, sizeof(header)) < 0) { + cmkem_errmsg("failed to read file '%s'.\n", real_path); + close(fd); + return CMKEM_READ_FILE_ERR; + } + *content_len = atoi(header); + + if (*content_len > buf_len) { + cmkem_errmsg("the header of file '%s' is invalid.\n", real_path); + close(fd); + return CMKEM_READ_FILE_ERR; + } + + if (read(fd, buf, *content_len) < 0) { + cmkem_errmsg("failed to read from file '%s'.\n", real_path); + close(fd); + return CMKEM_READ_FILE_ERR; + } + + close(fd); + return CMKEM_SUCCEED; +} + +CmkemErrCode read_iv_and_salt(const char *rand_path, unsigned char *iv, size_t iv_len, unsigned char *salt, + size_t salt_len) +{ + unsigned char iv_salt_buf[iv_len + salt_len] = {0}; + size_t iv_salt_len = 0; + CmkemErrCode ret = CMKEM_SUCCEED; + + ret = read_content_from_file(rand_path, iv_salt_buf, sizeof(iv_salt_buf), &iv_salt_len); + if (ret != CMKEM_SUCCEED) { + return ret; + } else if (iv_salt_len < sizeof(iv_salt_buf)) { + return CMKEM_READ_FILE_ERR; + } + + for (size_t i = 0; i < iv_len; i++) { + iv[i] = iv_salt_buf[i]; + } + for (size_t i = 0; i < salt_len; i++) { + salt[i] = iv_salt_buf[iv_len + i]; + } + + return CMKEM_SUCCEED; +} + +/* + * cmk plain is encrypted byfore stored. + * the iv_and_salt are used to derive a key to encrypt cmk plain, as well as decrypt cmk cipher + */ +static CmkemErrCode read_rand_and_drive_key(const char *rand_path, CmkemUStr **drived_key) +{ + unsigned char iv[KEY_METERIAL_LEN] = {0}; + unsigned char salt[KEY_METERIAL_LEN] = {0}; + CmkemUStr *reg_drived_key = NULL; + CmkemErrCode ret = CMKEM_SUCCEED; + + ret = read_iv_and_salt(rand_path, iv, sizeof(iv), salt, sizeof(salt)); + if (ret != CMKEM_SUCCEED) { + return ret; + } + + reg_drived_key = malloc_cmkem_ustr(DRIVED_KEY_LEN); + if (reg_drived_key == NULL) { + return CMKEM_MALLOC_MEM_ERR; + } + + if (PKCS5_PBKDF2_HMAC((const char *)iv, sizeof(iv), salt, sizeof(salt), ITERATE_ROUD, EVP_sha256(), + DRIVED_KEY_LEN, reg_drived_key->ustr_val) != 1) { + free_cmkem_ustr(reg_drived_key); + return CMKEM_DERIVED_KEY_ERR; + } + + *drived_key = reg_drived_key; + return CMKEM_SUCCEED; +} + +/* + * cmk is stored as cihper, so, rather then read it directly, we will : + * 1. read rand file firstly (rand_file_path = cmk_cipher_file_path.rand) + * 2. derive a key to decrypt cmk cipher + * 3. now, we finally get cmk plain + */ +static CmkemErrCode read_cmk_plain(const char *real_cmk_path, unsigned char *cmk_plain, size_t *plain_len) +{ + char rand_file_path[PATH_MAX] = {0}; + CmkemUStr *derivied_key = NULL; + unsigned char cmk_cipher[RSA3072_KEN_LEN] = {0}; + size_t cmk_cipher_len = 0; + int tmp_plainlen = 0; + CmkemErrCode ret = CMKEM_SUCCEED; + errno_t rc = 0; + + rc = sprintf_s(rand_file_path, PATH_MAX, "%s.rand", real_cmk_path); + securec_check_ss_c(rc, "", ""); + + ret = read_rand_and_drive_key(rand_file_path, &derivied_key); + if (ret != CMKEM_SUCCEED) { + free_cmkem_ustr(derivied_key); + return ret; + } + + ret = read_content_from_file(real_cmk_path, cmk_cipher, sizeof(cmk_cipher), &cmk_cipher_len); + if (ret != CMKEM_SUCCEED) { + free_cmkem_ustr(derivied_key); + return ret; + } + + AeadAesHamcEncKey derived_aead_key = AeadAesHamcEncKey(derivied_key->ustr_val, DRIVED_KEY_LEN); + free_cmkem_ustr(derivied_key); + tmp_plainlen = decrypt_data(cmk_cipher, cmk_cipher_len, derived_aead_key, cmk_plain, AEAD_AES_256_CBC_HMAC_SHA256); + if (tmp_plainlen <= 0) { + return CMKEM_DEC_CMK_ERR; + } + *plain_len = (size_t)tmp_plainlen; + + return CMKEM_SUCCEED; +} + +CmkemErrCode read_and_decrypt_cmk(const char *key_path, AsymmetricKeyType key_type, CmkemUStr **key_palin) +{ + CmkemErrCode ret = CMKEM_SUCCEED; + char key_file_path[PATH_MAX] = {0}; + CmkemUStr *ret_key_plain = NULL; + + if (key_type == PUBLIC_KEY) { + get_file_path_from_cmk_id(key_path, PUB_KEY_FILE, key_file_path, PATH_MAX); + } else { + get_file_path_from_cmk_id(key_path, PRIV_KEY_FILE, key_file_path, PATH_MAX); + } + + ret_key_plain = malloc_cmkem_ustr(MAX_ASYMM_KEY_BUF_LEN); + if (ret_key_plain == NULL) { + return CMKEM_MALLOC_MEM_ERR; + } + + ret = read_cmk_plain(key_file_path, ret_key_plain->ustr_val, &ret_key_plain->ustr_len); + if (ret != CMKEM_SUCCEED) { + free_cmkem_ustr_with_erase(ret_key_plain); + return ret; + } + + *key_palin = ret_key_plain; + return CMKEM_SUCCEED; +} \ No newline at end of file diff --git a/src/gausskernel/security/keymgr/localkms/security_localkms.cpp b/src/gausskernel/security/keymgr/localkms/security_localkms.cpp new file mode 100644 index 000000000..69f8ae440 --- /dev/null +++ b/src/gausskernel/security/keymgr/localkms/security_localkms.cpp @@ -0,0 +1,589 @@ +/* + * Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * security_localkms.cpp + * localkms is a lightweight key management component. + * different from other key management tools and services, localkms automatically generates CMKE when CREATE CMKO, + * it cannot manage key entities independently. + * localkms use openssl to generate cmk, then encrypt cmk plain and store cmk cipher in file. + * at the same time, we need to store/read the iv and salt that are used to derive a key to + * encrypt/decrypt cmk plain. + * + * IDENTIFICATION + * src/gausskernel/security/keymgr/src/localkms/security_localkms.cpp + * + * ------------------------------------------------------------------------- + */ + +#include "keymgr/localkms/security_localkms.h" +#include +#include +#include +#include +#include +#include +#include "securec.h" +#include "securec_check.h" +#include "keymgr/comm/security_utils.h" +#include "keymgr/localkms/security_cmkem_comm_algorithm.h" +#include "keymgr/localkms/security_file_enc.h" +#include "keymgr/encrypt/security_sm2_enc_key.h" +#include "keymgr/security_key_mgr.h" + +const int MAX_KEY_PATH_LEN = 64; +const int MIN_KEY_PATH_LEN = 1; + +static const char *g_support_algo[] = {"RSA_3072", "SM2", NULL}; + +LocalKmsMgr *localkms_new(KmErr *err) +{ + LocalKmsMgr *kms = (LocalKmsMgr *)km_alloc_zero(sizeof(LocalKmsMgr)); + if (kms == NULL) { + return NULL; + } + + kms->kmgr.err = err; + CmkemErrCode ret = set_global_env_value(); + if (ret != CMKEM_SUCCEED) { + km_free(kms); + km_err_msg(err, "%s", get_cmkem_errmsg(ret)); + return NULL; + } + kms->getpath = 1; + + return kms; +} + +void localkms_free(LocalKmsMgr *kms) +{ + if (kms == NULL) { + return; + } + + km_free(kms); +} + +/* + * check the KEY_PATH value in SQL : + * CREATE CLIENT MASTER KEY xxx (KEY_STORE = localkms, KEY_PATH = xxx, ...) + * only support 0-9, a-z, A_Z, '_', '.', '-' + */ +static CmkemErrCode check_key_path_value(const char *key_path_value) +{ + char cur_char = 0; + const char legal_symbol[] = {'_', '.', '-'}; + + for (size_t i = 0; i < strlen(key_path_value); i++) { + cur_char = key_path_value[i]; + if (cur_char >= '0' && cur_char <= '9') { + continue; + } else if (cur_char >= 'A' && cur_char <= 'Z') { + continue; + } else if (cur_char >= 'a' && cur_char <= 'z') { + continue; + } else if (strchr(legal_symbol, cur_char) != NULL) { + continue; + } else { + cmkem_errmsg("the key_path value '%s' contains invalid charachter '%c'.", key_path_value, cur_char); + return CMKEM_CHECK_CMK_ID_ERR; + } + } + + return CMKEM_SUCCEED; +} + + +/* + * check the $key_path_value in SQL : + * CREATE CLIENT MASTER KEY xxx (KEY_STORE = localkms, KEY_PATH = $key_path_value, ...) + * 1. is the length of $key_path_value in range (MIN_KEY_PATH_LEN, MAX_KEY_PATH_LEN) ? + * 2. does $key_path_value contain unsupport char ? + * 3. is environment variable $LOCALKMS_FILE_PATH/$GAUSSHOME set ? + * 4. are the cmk and rand keys named according $key_path_value exist ? + */ +static CmkemErrCode check_cmk_id_validity(const char *keyid) +{ + char cmk_file_path[PATH_MAX] = {0}; + CmkemErrCode ret = CMKEM_SUCCEED; + LocalkmsFileType all_file[] = {PUB_KEY_FILE, PRIV_KEY_FILE, PUB_KEY_ENC_IV_FILE, PRIV_KEY_ENC_IV_FILE}; + + if (strlen(keyid) < MIN_KEY_PATH_LEN || strlen(keyid) > MAX_KEY_PATH_LEN) { + return CMKEM_CHECK_CMK_ID_ERR; + } + + ret = check_key_path_value(keyid); + if (ret != CMKEM_SUCCEED) { + return ret; + } + + ret = set_global_env_value(); + if (ret != CMKEM_SUCCEED) { + return ret; + } + + for (size_t i = 0; i < sizeof(all_file) / sizeof(all_file[0]); i++) { + get_file_path_from_cmk_id(keyid, all_file[i], cmk_file_path, PATH_MAX); + ret = check_file_exist(cmk_file_path, CREATE_KEY_FILE); + if (ret != CMKEM_SUCCEED) { + return ret; + } + } + + return CMKEM_SUCCEED; +} + +/* + * for now, localkms only supports asymmetric cryptographic algorithm : + * 1. RSA_2048 (not safe now, please use 3072) + * 2. RSA_3072 + * 3. SM3 (for legal reasons, it is only supported in China) + */ +static CmkemErrCode check_cmk_algo_validity(const char *cmk_algo) +{ + char error_msg_buf[MAX_CMKEM_ERRMSG_BUF_SIZE] = {0}; + error_t rc = 0; + + for (size_t i = 0; g_support_algo[i] != NULL; i++) { + if (strcasecmp(cmk_algo, g_support_algo[i]) == 0) { + return CMKEM_SUCCEED; + } + } + + rc = sprintf_s(error_msg_buf, MAX_CMKEM_ERRMSG_BUF_SIZE, "unspported algorithm '%s', localkms only support: ", + cmk_algo); + securec_check_ss_c(rc, "", ""); + + for (size_t i = 0; g_support_algo[i] != NULL; i++) { + rc = strcat_s(error_msg_buf, MAX_CMKEM_ERRMSG_BUF_SIZE, g_support_algo[i]); + securec_check_c(rc, "\0", "\0"); + rc = strcat_s(error_msg_buf, MAX_CMKEM_ERRMSG_BUF_SIZE, " "); + securec_check_c(rc, "\0", "\0"); + } + + cmkem_errmsg("%s", error_msg_buf); + return CMKEM_CHECK_ALGO_ERR; +} + +/* + * rsa keys generated by openssl::RSA_generate_key_ex() are RSA type, + * so, we will temporarily write to the bio buffer at first, + * then, read out and stored as strings from the bio buffer + */ +static CmkemErrCode write_rsa_keypair_to_file(RSA *rsa_key_pair, const char *cmk_path) +{ + BIO *tmp_pub_key = NULL; + BIO *tmp_priv_key = NULL; + CmkemUStr *rsa_pub_key = NULL; + CmkemUStr *rsa_priv_key = NULL; + char cmk_pub_file_path[PATH_MAX] = {0}; + char cmk_priv_file_path[PATH_MAX] = {0}; + CmkemErrCode ret = CMKEM_SUCCEED; + + ret = write_rsa_keypair_to_biobuf(rsa_key_pair, &tmp_pub_key, &tmp_priv_key); + if (ret != CMKEM_SUCCEED) { + return ret; + } + + ret = read_rsa_key_from_biobuf(tmp_pub_key, &rsa_pub_key); + BIO_free(tmp_pub_key); + if (ret != CMKEM_SUCCEED) { + BIO_free(tmp_priv_key); + return ret; + } + + ret = read_rsa_key_from_biobuf(tmp_priv_key, &rsa_priv_key); + BIO_free(tmp_priv_key); + if (ret != CMKEM_SUCCEED) { + free_cmkem_ustr_with_erase(rsa_pub_key); + return ret; + } + + get_file_path_from_cmk_id(cmk_path, PUB_KEY_FILE, cmk_pub_file_path, PATH_MAX); + get_file_path_from_cmk_id(cmk_path, PRIV_KEY_FILE, cmk_priv_file_path, PATH_MAX); + + ret = encrypt_and_write_key(cmk_pub_file_path, rsa_pub_key); + free_cmkem_ustr_with_erase(rsa_pub_key); + if (ret != CMKEM_SUCCEED) { + free_cmkem_ustr_with_erase(rsa_priv_key); + return ret; + } + + ret = encrypt_and_write_key(cmk_priv_file_path, rsa_priv_key); + free_cmkem_ustr_with_erase(rsa_priv_key); + + return ret; +} + +static CmkemErrCode create_and_write_rsa_key_pair(const char *cmk_id, size_t rsa_key_len) +{ + CmkemErrCode ret = CMKEM_SUCCEED; + RSA *rsa_key_pair = NULL; + + rsa_key_pair = create_rsa_keypair(rsa_key_len); + if (rsa_key_pair == NULL) { + return CMKEM_GEN_RSA_KEY_ERR; + } + + ret = write_rsa_keypair_to_file(rsa_key_pair, cmk_id); + RSA_free(rsa_key_pair); + check_cmkem_ret(ret); + + return CMKEM_SUCCEED; +} + +static CmkemErrCode write_sm2_keypair_to_file(Sm2KeyPair *sm2_key_pair, const char *cmk_path) +{ + char cmk_pub_file_path[PATH_MAX] = {0}; + char cmk_priv_file_path[PATH_MAX] = {0}; + CmkemErrCode ret = CMKEM_SUCCEED; + + get_file_path_from_cmk_id(cmk_path, PUB_KEY_FILE, cmk_pub_file_path, PATH_MAX); + get_file_path_from_cmk_id(cmk_path, PRIV_KEY_FILE, cmk_priv_file_path, PATH_MAX); + + ret = encrypt_and_write_key(cmk_pub_file_path, sm2_key_pair->pub_key); + check_cmkem_ret(ret); + + ret = encrypt_and_write_key(cmk_priv_file_path, sm2_key_pair->priv_key); + check_cmkem_ret(ret); + + return CMKEM_SUCCEED; +} + +static CmkemErrCode create_and_write_sm2_key_pair(const char *cmk_id) +{ + CmkemErrCode ret = CMKEM_SUCCEED; + Sm2KeyPair *sm2_key_pair = NULL; + + sm2_key_pair = generate_encrypt_pair_key(); + if (sm2_key_pair == NULL) { + return CMKEM_GEN_SM2_KEY_ERR; + } + + ret = write_sm2_keypair_to_file(sm2_key_pair, cmk_id); + free_cmkem_ustr_with_erase(sm2_key_pair->priv_key); + free_cmkem_ustr_with_erase(sm2_key_pair->pub_key); + km_free(sm2_key_pair); + check_cmkem_ret(ret); + + return CMKEM_SUCCEED; +} + +CmkemErrCode read_and_decrypt_bio_cmk(const char *key_path, AsymmetricKeyType key_type, BIO **key_plain) +{ + CmkemErrCode ret = CMKEM_SUCCEED; + int bio_write_len = 0; + CmkemUStr *tmp_key_plain = NULL; + BIO *ret_key_plain = NULL; + + ret = read_and_decrypt_cmk(key_path, key_type, &tmp_key_plain); + check_cmkem_ret(ret); + + ret_key_plain = BIO_new(BIO_s_mem()); + if (ret_key_plain == NULL) { + free_cmkem_ustr_with_erase(tmp_key_plain); + return CMKEM_MALLOC_MEM_ERR; + } + + bio_write_len = BIO_write(ret_key_plain, tmp_key_plain->ustr_val, tmp_key_plain->ustr_len); + free_cmkem_ustr_with_erase(tmp_key_plain); + if (bio_write_len < 0) { + return CMKEM_WRITE_TO_BIO_ERR; + } + + *key_plain = ret_key_plain; + return CMKEM_SUCCEED; +} + +static CmkemErrCode encrypt_cek_with_rsa(CmkemUStr *cek_plain, const char *cmk_id_str, CmkemUStr **cek_cipher) +{ + CmkemErrCode ret = CMKEM_SUCCEED; + BIO *cmk_plain = NULL; + RSA *rsa_cmk_plain = NULL; + CmkemUStr *ret_cek_cipher = NULL; + int enc_ret = 0; + + ret = read_and_decrypt_bio_cmk(cmk_id_str, PUBLIC_KEY, &cmk_plain); + check_cmkem_ret(ret); + + rsa_cmk_plain = PEM_read_bio_RSA_PUBKEY(cmk_plain, NULL, NULL, NULL); + BIO_free(cmk_plain); + if (rsa_cmk_plain == NULL) { + return CMKEM_READ_FROM_BIO_ERR; + } + + ret_cek_cipher = malloc_cmkem_ustr(MAX_ASYMM_KEY_BUF_LEN); + if (ret_cek_cipher == NULL) { + RSA_free(rsa_cmk_plain); + return CMKEM_MALLOC_MEM_ERR; + } + + enc_ret = RSA_public_encrypt(cek_plain->ustr_len, cek_plain->ustr_val, ret_cek_cipher->ustr_val, rsa_cmk_plain, + RSA_PKCS1_OAEP_PADDING); + RSA_free(rsa_cmk_plain); + if (enc_ret == -1) { + free_cmkem_ustr(ret_cek_cipher); + return CMKEM_RSA_ENCRYPT_ERR; + } + + ret_cek_cipher->ustr_len = (size_t)enc_ret; + *cek_cipher = ret_cek_cipher; + + return CMKEM_SUCCEED; +} + +static CmkemErrCode decrypt_cek_with_rsa(CmkemUStr *cek_cipher, const char *cmk_id_str, CmkemUStr **cek_plain) +{ + CmkemErrCode ret = CMKEM_SUCCEED; + BIO *cmk_plain = NULL; + RSA *rsa_cmk_plain = NULL; + CmkemUStr *ret_cek_plain = NULL; + int enc_ret = 0; + + ret = read_and_decrypt_bio_cmk(cmk_id_str, PRIVATE_KEY, &cmk_plain); + check_cmkem_ret(ret); + + rsa_cmk_plain = PEM_read_bio_RSAPrivateKey(cmk_plain, NULL, NULL, NULL); + BIO_free(cmk_plain); + if (rsa_cmk_plain == NULL) { + return CMKEM_READ_FROM_BIO_ERR; + } + + ret_cek_plain = malloc_cmkem_ustr(MAX_ASYMM_KEY_BUF_LEN); + if (ret_cek_plain == NULL) { + RSA_free(rsa_cmk_plain); + return CMKEM_MALLOC_MEM_ERR; + } + + enc_ret = RSA_private_decrypt(cek_cipher->ustr_len, cek_cipher->ustr_val, ret_cek_plain->ustr_val, rsa_cmk_plain, + RSA_PKCS1_OAEP_PADDING); + RSA_free(rsa_cmk_plain); + if (enc_ret == -1) { + free_cmkem_ustr_with_erase(ret_cek_plain); + return CMKEM_RSA_DECRYPT_ERR; + } + + ret_cek_plain->ustr_len = (size_t) enc_ret; + *cek_plain = ret_cek_plain; + + return CMKEM_SUCCEED; +} + +static CmkemErrCode encrypt_cek_with_sm2(CmkemUStr *cek_plain, const char *cmk_id_str, CmkemUStr **cek_cipher) +{ + CmkemErrCode ret = CMKEM_SUCCEED; + CmkemUStr *cmk_plain = NULL; + + ret = read_and_decrypt_cmk(cmk_id_str, PUBLIC_KEY, &cmk_plain); + check_cmkem_ret(ret); + + ret = encrypt_with_sm2_pubkey(cek_plain, cmk_plain, cek_cipher); + free_cmkem_ustr_with_erase(cmk_plain); + check_cmkem_ret(ret); + + return CMKEM_SUCCEED; +} + +static CmkemErrCode decrypt_cek_with_sm2(CmkemUStr *cek_cipher, const char *cmk_id_str, CmkemUStr **cek_plain) +{ + CmkemErrCode ret = CMKEM_SUCCEED; + CmkemUStr *cmk_plain = NULL; + + ret = read_and_decrypt_cmk(cmk_id_str, PRIVATE_KEY, &cmk_plain); + check_cmkem_ret(ret); + + ret = decrypt_with_sm2_privkey(cek_cipher, cmk_plain, cek_plain); + free_cmkem_ustr_with_erase(cmk_plain); + check_cmkem_ret(ret); + + return CMKEM_SUCCEED; +} + +KeyMgr *kms_new(KmErr *err) +{ + return (KeyMgr *)localkms_new(err); +} + +void kms_free(KeyMgr *kmgr) +{ + localkms_free((LocalKmsMgr *)kmgr); +} + +void kms_mk_create(KeyMgr *kmgr, KeyInfo info) +{ + LocalKmsMgr *kms = (LocalKmsMgr *)(void *)kmgr; + CmkemErrCode ret = CMKEM_UNKNOWN_ERR; + + switch (get_algo_by_str(info.algo)) { + case AT_RSA_2048: + km_err_msg(kms->kmgr.err, "rsa_2048 is not safe now, please use rsa_3072 instead."); + return; + case AT_RSA_3072: + ret = create_and_write_rsa_key_pair(info.id, RSA3072_KEN_LEN); + break; + case AT_SM2: + ret = create_and_write_sm2_key_pair(info.id); + break; + default: + break; + } + + if (ret != CMKEM_SUCCEED) { + km_err_msg(kms->kmgr.err, "%s", get_cmkem_errmsg(ret)); + } +} + +static void rm_file(const char *file) +{ + if (remove(file) != 0) { + (void)printf("failed to remove file '%s'.\n", file); + } +} + +void kms_mk_delete(KeyMgr *kmgr, KeyInfo info) +{ + char pub_cmk_file[PATH_MAX] = {0}; + char priv_cmk_file[PATH_MAX] = {0}; + errno_t rc = 0; + + get_file_path_from_cmk_id(info.id, PUB_KEY_FILE, pub_cmk_file, PATH_MAX); + get_file_path_from_cmk_id(info.id, PRIV_KEY_FILE, priv_cmk_file, PATH_MAX); + + rm_file(pub_cmk_file); + rm_file(priv_cmk_file); + + rc = strcat_s(pub_cmk_file, PATH_MAX, ".rand"); + securec_check_c(rc, "", ""); + rc = strcat_s(priv_cmk_file, PATH_MAX, ".rand"); + securec_check_c(rc, "", ""); + + rm_file(pub_cmk_file); + rm_file(priv_cmk_file); +} + +char *kms_mk_select(KeyMgr *kmgr, KeyInfo info) +{ + LocalKmsMgr *kms = (LocalKmsMgr *)(void *)kmgr; + CmkemErrCode ret = CMKEM_UNKNOWN_ERR; + + if (info.id == NULL) { + km_err_msg(kms->kmgr.err, "failed to create client master key, failed to find arg: KEY_PATH."); + return NULL; + } + + if (info.algo == NULL) { + km_err_msg(kms->kmgr.err, "failed to create client master key, failed to find arg: ALGORITHM."); + return NULL; + } + + ret = check_cmk_id_validity(info.id); + if (ret != CMKEM_SUCCEED) { + km_err_msg(kms->kmgr.err, "%s", get_cmkem_errmsg(ret)); + return NULL; + } + + if (strcasecmp(info.algo, "RSA_2048") == 0) { + km_err_msg(kms->kmgr.err, "rsa_2048 is not safe now, please use rsa_3072 instead."); + return NULL; + } + + ret = check_cmk_algo_validity(info.algo); + if (ret != CMKEM_SUCCEED) { + km_err_msg(kms->kmgr.err, "%s", get_cmkem_errmsg(ret)); + return NULL; + } + + return strdup("active"); +} + +KmUnStr kms_mk_encrypt(KeyMgr *kmgr, KeyInfo info, KmUnStr plain) +{ + LocalKmsMgr *kms = (LocalKmsMgr *)(void *)kmgr; + CmkemErrCode ret = CMKEM_UNKNOWN_ERR; + KmUnStr cipher = {0}; + + CmkemUStr _plain = {plain.val, plain.len}; + CmkemUStr *_cipher = NULL; + + switch (get_algo_by_str(info.algo)) { + case AT_RSA_2048: + km_err_msg(kms->kmgr.err, "the algorithm of master key is rsa_2048, but rsa_2048 is not safe now, " + "please create new master key with rsa_3072."); + return cipher; + case AT_RSA_3072: + ret = encrypt_cek_with_rsa(&_plain, info.id, &_cipher); + break; + case AT_SM2: + ret = encrypt_cek_with_sm2(&_plain, info.id, &_cipher); + break; + default: + break; + } + + if (ret != CMKEM_SUCCEED) { + km_err_msg(kms->kmgr.err, "%s", get_cmkem_errmsg(ret)); + return cipher; + } + + cipher.val = _cipher->ustr_val; + cipher.len = _cipher->ustr_len; + return cipher; +} + +KmUnStr kms_mk_decrypt(KeyMgr *kmgr, KeyInfo info, KmUnStr cipher) +{ + LocalKmsMgr *kms = (LocalKmsMgr *)(void *)kmgr; + CmkemErrCode ret = CMKEM_UNKNOWN_ERR; + KmUnStr plain = {0}; + + CmkemUStr _cipher = {cipher.val, cipher.len}; + CmkemUStr *_plain = NULL; + + switch (get_algo_by_str(info.algo)) { + case AT_RSA_2048: /* only decrypt the existing data encrypted by old version of opengauss */ + case AT_RSA_3072: + ret = decrypt_cek_with_rsa(&_cipher, info.id, &_plain); + break; + case AT_SM2: + ret = decrypt_cek_with_sm2(&_cipher, info.id, &_plain); + break; + default: + break; + } + + if (ret != CMKEM_SUCCEED) { + km_err_msg(kms->kmgr.err, "%s", get_cmkem_errmsg(ret)); + return plain; + } + + plain.val = _plain->ustr_val; + plain.len = _plain->ustr_len; + return plain; +} + +KeyMethod localkms = { + "localkms", + + kms_new, /* kmgr_new */ + kms_free, /* kmgr_free */ + NULL, /* kmgr_set_arg */ + + kms_mk_create, /* mk_create */ + kms_mk_delete, /* mk_delete */ + kms_mk_select, /* mk_select */ + kms_mk_encrypt, /* mk_encrypt */ + kms_mk_decrypt, /* mk_decrypt */ + + NULL, /* dk_create */ +}; \ No newline at end of file diff --git a/src/include/Makefile b/src/include/Makefile index be8dbbc48..aec88fc06 100644 --- a/src/include/Makefile +++ b/src/include/Makefile @@ -27,7 +27,7 @@ SUBDIRS = bootstrap catalog commands client_logic datatype executor foreign gs_p storage vecexecutor access streaming access/obs \ hotpatch \ gstrace \ - knl knl/knl_guc threadpool workload lite + knl knl/knl_guc threadpool workload lite keymgr ifneq ($(enable_multiple_nodes), yes) SUBDIRS += storage/smgr storage/lock storage/buf storage/item storage/cstore storage/mot fmgr diff --git a/src/include/keymgr/comm/security_aksk.h b/src/include/keymgr/comm/security_aksk.h new file mode 100644 index 000000000..f496eb86a --- /dev/null +++ b/src/include/keymgr/comm/security_aksk.h @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * security_aksk.h + * + * IDENTIFICATION + * src/gausskernel/security/keymgr/include/comm/security_aksk.h + * + * ------------------------------------------------------------------------- + */ + +#ifndef KM_AKSK_H_ +#define KM_AKSK_H_ + +#include + +#define KM_MAX_HTTP_HDR_NUM 16 + +typedef struct { + const char *ak; + const char *sk; + + /* http header */ + const char *method; + const char *host; + const char *uri; + + /* http headers */ + const char *reqhdrs[KM_MAX_HTTP_HDR_NUM]; + size_t hdrcnt; + + /* http body */ + const char *reqbody; +} AkSkPara; + +typedef struct { + char *host; + char *date; + char *signature; +} AkSkSign; + +void km_aksk_add_reqhdr(AkSkPara *para, const char *hdr); +AkSkSign km_aksk_sign(AkSkPara *para); + +#endif \ No newline at end of file diff --git a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/reg_cmkem_manager_main.h b/src/include/keymgr/comm/security_encode.h similarity index 55% rename from src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/reg_cmkem_manager_main.h rename to src/include/keymgr/comm/security_encode.h index 5ebf0e9a4..9ce03e675 100644 --- a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/reg_cmkem_manager_main.h +++ b/src/include/keymgr/comm/security_encode.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * Copyright (c) 2022 Huawei Technologies Co.,Ltd. * * openGauss is licensed under Mulan PSL v2. * You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -13,23 +13,26 @@ * See the Mulan PSL v2 for more details. * ------------------------------------------------------------------------- * - * registre_hook_func.h - * you can register your own key management tool/component/service here. - * what's more, you can choose which to use according to the version and usage scenarios. - * + * security_encode.h + * * IDENTIFICATION - * src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/registre_hook_func.h + * src/gausskernel/security/keymgr/include/comm/security_encode.h * * ------------------------------------------------------------------------- */ -#ifndef REG_HOOK_FUNC_H -#define REG_HOOK_FUNC_H +#ifndef _KM_ENCODE_H_ +#define _KM_ENCODE_H_ #include -#include "cmkem_comm.h" +#include "keymgr/comm/security_utils.h" + +KmStr km_hex_encode(KmUnStr str); +KmUnStr km_hex_decode(KmStr hex); +KmStr km_sha_encode(KmUnStr str); +KmUnStr km_sha_decode(KmStr shahex); -extern CmkemErrCode reg_all_cmk_entity_manager(); -extern void exit_all_cmk_entity_manager(); +size_t km_url_encode(const char *in, char *out, size_t outsz); +size_t km_hex_sha(const char *data, size_t datalen, char *hexbuf, size_t bufsz); -#endif /* REGISTER_HOOK_FUNC_H */ +#endif \ No newline at end of file diff --git a/src/include/keymgr/comm/security_error.h b/src/include/keymgr/comm/security_error.h new file mode 100644 index 000000000..103df7b81 --- /dev/null +++ b/src/include/keymgr/comm/security_error.h @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * security_error.h + * + * IDENTIFICATION + * src/gausskernel/security/keymgr/include/comm/security_error.h + * + * ------------------------------------------------------------------------- + */ + +#ifndef _KM_ERROR_H_ +#define _KM_ERROR_H_ + +#include +#include +#include "securec.h" +#include "securec_check.h" + +#ifdef FRONTEND /* used by libpq */ +#define km_securec_check_ss securec_check_ss_c +#define km_securec_check securec_check_c + +#define km_alloc malloc +#define km_alloc_zero(sz) calloc(1, (sz)) +#define km_cxt_alloc_zero(cxt, sz) km_alloc_zero((sz)) +#define km_strdup(str) ((str) == NULL ? NULL : strdup((str))) +#define km_strndup strndup +#define km_safe_free(ptr) \ + do { \ + if ((ptr) != NULL) { \ + free((ptr)); \ + (ptr) = NULL; \ + } \ + } while (0) +#define km_free free + +#else /* used by kernel */ +#include "utils/palloc.h" +#define km_securec_check_ss securec_check_ss +#define km_securec_check securec_check + +#define km_alloc palloc +#define km_alloc_zero(sz) palloc0((sz)) +void *km_cxt_alloc_zero(MemoryContext cxt, size_t size); +#define km_strdup pstrdup +#define km_strndup pnstrdup +#define km_safe_free FREE_POINTER +#define km_free pfree +#endif /* end */ + +#define KM_EREASE_DATA(buf, bufsz, datasz) \ + do { \ + if ((buf) != NULL) { \ + errno_t _grc = memset_s((buf), (bufsz), 0, (datasz)); \ + km_securec_check(_grc, "", ""); \ + } \ + } while (0) + +typedef enum { + KM_ERR_INIT = 0, + KM_ERR_MEMORY, +} KmErrCode; + +typedef struct { + char *buf; + size_t sz; /* max buffer len */ + KmErrCode code; + +#ifdef FRONTEND + char cxt; /* not used */ +#else + MemoryContext cxt; +#endif +} KmErr; + +#define km_err_code(err, code) ((err)->code = (code)) +#define km_err_msg(err, fmt, ...) \ + do { \ + errno_t grc; \ + if ((err) != NULL && (err)->buf != NULL) { \ + grc = sprintf_s((err)->buf, (err)->sz, (fmt), ##__VA_ARGS__); \ + km_securec_check_ss(grc, "", ""); \ + } \ + } while (0) +#define km_err_reset(err) \ + do { \ + errno_t grc; \ + grc = memset_s((err)->buf, (err)->sz, 0, (err)->sz); \ + km_securec_check(grc, "", ""); \ + (err)->code = KM_ERR_INIT; \ + } while (0) +#define km_err_get_msg(err) (err)->buf +#define km_err_catch(err) (strlen((err)->buf) > 0 || (err)->code != KM_ERR_INIT) + +KmErr *km_err_new(size_t sz); +void km_err_free(KmErr *err); + +#endif \ No newline at end of file diff --git a/src/include/keymgr/comm/security_http.h b/src/include/keymgr/comm/security_http.h new file mode 100644 index 000000000..a660dfab0 --- /dev/null +++ b/src/include/keymgr/comm/security_http.h @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * security_http.h + * + * IDENTIFICATION + * src/gausskernel/security/keymgr/include/comm/security_http.h + * + * ------------------------------------------------------------------------- + */ + +#ifndef _KM_HTTP_H_ +#define _KM_HTTP_H_ + +#include +#include "keymgr/comm/security_httpscan.h" + +#define HTTP_MIN_UNEXCEPTED_CODE 300 + +typedef struct { + CURL *client; + + /* http config */ + int timeout; + int sslverify; + + /* http request line (save for error reporting) */ + char *url; + CURLoption method; + /* http request header */ + struct curl_slist *reqhdrs; + char *reqlen; /* store request header 'Content-Length: ...', it's necessary for some http server */ + const char *reqbody; + + ResScan *resscan; + + /* http response line */ + long state; + /* http response header */ + char* reshdrs; + /* http response body: we store them in 'resscan' */ + + /* if use aksk, generate signature of http message */ + char *ak; + char *sk; + + KmErr *err; +} HttpMgr; + +HttpMgr *httpmgr_new(KmErr *errbuf); +void httpmgr_free(HttpMgr *mgr); + +#define httpmgr_ak_set(http, akval) (http)->ak = km_strdup((akval)) +#define httpmgr_sk_set(http, skval) (http)->sk = km_strdup((skval)) + +void httpmgr_set_req_line(HttpMgr *mgr, const char *url, CURLoption method, const char *cacert); +void httpmgr_set_req_header(HttpMgr *mgr, const char *header); +void httpmgr_set_req_body(HttpMgr *mgr, char *reqbody); +char *httpmgr_get_req_body_len(HttpMgr *http, char *reqbody); + +void httpmgr_set_response(HttpMgr *mgr, char remain[3], const char *hdrkey); +void httpmgr_receive(HttpMgr *mgr); +#ifdef ENABLE_KM_DEBUG +#define httpmgr_set_output(mgr, file) resscan_set_output((mgr)->resscan, (file)) +#endif + +#define httpmgr_get_status(http) (http)->state +#define httpmgr_get_res_header(http) (http)->resscan->header +#define httpmgr_get_res_body(http) (http)->resscan->body + +#endif \ No newline at end of file diff --git a/src/include/keymgr/comm/security_httpscan.h b/src/include/keymgr/comm/security_httpscan.h new file mode 100644 index 000000000..e76416d22 --- /dev/null +++ b/src/include/keymgr/comm/security_httpscan.h @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * security_httpscan.h + * + * IDENTIFICATION + * src/gausskernel/security/keymgr/include/comm/security_httpscan.h + * + * ------------------------------------------------------------------------- + */ + +#ifndef __KM_HTTPSCAN_H_ +#define __KM_HTTPSCAN_H_ + +#include +#include +#include "keymgr/comm/security_error.h" + +typedef enum { + RS_INIT = 0, + RS_END, + RS_ENDEARLY, + RS_ERROR, + + RS_LINE, + RS_HEADER, + RS_BODY, +} ResState; + +/* + * state machine: + * RS_INIT + * | RA_START + * RS_LINE <-- RA_START + * | RA_TEXT + * RS_HEADER <-- RA_TEXT + * | RA_NEWLINE + * RS_BODY <-- RA_TEXT + * | RA_ZERO/RA_NEWLINE + * RS_END + * + * see ScanLine. + */ + +typedef enum { + RA_START, /* start with ‘HTTP/’ */ + RA_NEWLINE, /* /n */ + RA_ZERO, /* '0' */ + RA_TEXT, /* else */ + RA_NULL, +} ResAction; + +typedef enum { + RR_LINE = 0, + RR_HEADER, + RR_BODY, +} ResRemain; + +typedef struct { + ResState stat; + char remain[3]; /* ResponseLine, ResponseHeader, ResponseBody */ + + char *line; + char *header; /* for now, we just need 1 header, so we don't store all headers in a list. */ + char *body; + size_t bodylen; + + char *hdrkey; /* the header we need */ + + char *file; /* if != NULL, will echo all http response message into the file */ + FILE *fd; + + KmErr *err; +} ResScan; + +ResScan *resscan_new(KmErr *errbuf); +void resscan_free(ResScan *scan); +void resscan_set_receive(ResScan *scan, char remain[3], const char *hdrkey); +void resscan_line(ResScan *scan, const char *line, size_t linelen); +#ifdef ENABLE_KM_DEBUG +void resscan_set_output(ResScan *scan, const char *file); +void resscan_output(ResScan *scan, const char *line); +void resscan_finish(ResScan *scan); +#endif +#endif \ No newline at end of file diff --git a/src/include/keymgr/comm/security_json.h b/src/include/keymgr/comm/security_json.h new file mode 100644 index 000000000..bff5621bc --- /dev/null +++ b/src/include/keymgr/comm/security_json.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * security_json.h + * + * IDENTIFICATION + * src/gausskernel/security/keymgr/include/comm/security_json.h + * + * ------------------------------------------------------------------------- + */ + +#ifndef __KM_JSON_H_ +#define __KM_JSON_H_ + +typedef struct { + const char *src; + const char *dest; +} ReplaceRule; + +#define JS_TO_STR(R) #R + +char *json_replace(const char *json, ReplaceRule rules[], int rulecnt); +char *json_find(const char *json, const char *key); + +#endif \ No newline at end of file diff --git a/src/include/keymgr/comm/security_utils.h b/src/include/keymgr/comm/security_utils.h new file mode 100644 index 000000000..5b3760813 --- /dev/null +++ b/src/include/keymgr/comm/security_utils.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * security_utils.h + * + * IDENTIFICATION + * src/gausskernel/security/keymgr/include/comm/security_utils.h + * + * ------------------------------------------------------------------------- + */ + +#ifndef __KM_UTILS_H_ +#define __KM_UTILS_H_ + +#include +#include "securec.h" +#include "securec_check.h" +#include "keymgr/comm/security_error.h" + +/* string utils */ +typedef struct { + unsigned char *val; + size_t len; +} KmUnStr; + +typedef struct { + char *val; + size_t len; +} KmStr; + +char *km_str_strip(char *str); +char *km_env_get(const char *env); +int km_str_start_with(const char *str, const char *start); + +/* array utils */ +#define ARR_LEN(arr) ((int)sizeof((arr)) / sizeof((arr)[0])) + +/* key-value utils */ +#define KV_BUF_SZ 3072 + + +typedef struct { + const char *input; + int inlen; + int curpos; /* current scan position */ + int kvcnt; + + char key[KV_BUF_SZ]; + char value[KV_BUF_SZ]; +} KvScan; + +KvScan *kv_scan_init(const char *input); +void kv_scan_drop(KvScan *scan); +int kv_scan_exec(KvScan *scan); +char *km_realpath(const char *path, KmErr *err); +const char *km_str_spilt(const char *data, char *buf, size_t bufsz); + +#endif \ No newline at end of file diff --git a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/cmkem_version_control.h b/src/include/keymgr/comm/security_version.h similarity index 68% rename from src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/cmkem_version_control.h rename to src/include/keymgr/comm/security_version.h index c67d1835c..46a1bbee5 100644 --- a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/cmkem_version_control.h +++ b/src/include/keymgr/comm/security_version.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * Copyright (c) 2022 Huawei Technologies Co.,Ltd. * * openGauss is licensed under Mulan PSL v2. * You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -13,28 +13,27 @@ * See the Mulan PSL v2 for more details. * ------------------------------------------------------------------------- * - * cmkem_version_control.h - * + * security_version.h + * * IDENTIFICATION - * src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/cmkem_version_control.h + * src/gausskernel/security/keymgr/include/comm/security_version.h * * ------------------------------------------------------------------------- */ -#ifndef CMKEM_VERSION_CONTROL_H -#define CMKEM_VERSION_CONTROL_H #include "pg_config.h" -#if ((defined(ENABLE_MULTIPLE_NODES)) || (defined(ENABLE_PRIVATEGAUSS) && (!defined(ENABLE_LITE_MODE)))) +#if ((defined(ENABLE_MULTIPLE_NODES)) || (defined(ENABLE_PRIVATEGAUSS))) +#ifdef ENABLE_KT #define ENABLE_GS_KTOOL +#endif + #define ENABLE_HUAWEI_KMS #ifdef ENABLE_UT -#define ENABLE_LOCAL_KMS -#endif /* ENABLE_UT */ - -#else -#define ENABLE_LOCAL_KMS +#define ENABLE_LOCALKMS #endif -#endif /* CMKEM_VERSION_CONTROL_H */ +#else +#define ENABLE_LOCALKMS +#endif \ No newline at end of file diff --git a/src/common/interfaces/libpq/client_logic_hooks/encryption_hooks/aead_aes_hamc_enc_key.h b/src/include/keymgr/encrypt/security_aead_aes_hamc_enc_key.h similarity index 91% rename from src/common/interfaces/libpq/client_logic_hooks/encryption_hooks/aead_aes_hamc_enc_key.h rename to src/include/keymgr/encrypt/security_aead_aes_hamc_enc_key.h index 8dc8d6d61..b66c0307d 100644 --- a/src/common/interfaces/libpq/client_logic_hooks/encryption_hooks/aead_aes_hamc_enc_key.h +++ b/src/include/keymgr/encrypt/security_aead_aes_hamc_enc_key.h @@ -13,10 +13,10 @@ * See the Mulan PSL v2 for more details. * ------------------------------------------------------------------------- * - * aead_aes_hamc_enc_key.h + * security_aead_aes_hamc_enc_key.h * * IDENTIFICATION - * src\common\interfaces\libpq\client_logic_hooks\encryption_hooks\aead_aes_hamc_enc_key.h + * src/gausskernel/security/keymgr/include/encrypt/security_aead_aes_hamc_enc_key.h * * ------------------------------------------------------------------------- */ @@ -26,7 +26,6 @@ #include "openssl/hmac.h" #include "openssl/ossl_typ.h" -#include "cipher.h" class HmacCtxGroup { public: @@ -50,13 +49,7 @@ class HmacCtxGroup { HMAC_CTX* ctx_worker; HMAC_CTX* ctx_template; private: - void free_hmac_ctx(HMAC_CTX** ctx_tmp) - { - if (*ctx_tmp != NULL) { - HMAC_CTX_free(*ctx_tmp); - *ctx_tmp = NULL; - } - } + void free_hmac_ctx(HMAC_CTX** ctx_tmp) const; }; /* @@ -85,12 +78,12 @@ class AeadAesHamcEncKey { HmacCtxGroup hmac_ctx_group_iv; /* store ctx to reuse for IV call */ HmacCtxGroup hmac_ctx_group_mac; /* store ctx to reuse for hmac generation or hmac check call */ + EVP_CIPHER_CTX *encrypt_ctx; /* store encrypt ctx to reuse EVP_CIPHER_CTX */ + EVP_CIPHER_CTX *decrypt_ctx; /* store encrypt ctx to reuse EVP_CIPHER_CTX */ private: bool HKDF(const unsigned char *key, int key_len, const unsigned char *data, int data_len, unsigned char *result); - -private: /* Encryption Key Salt format. This is used to derive the encryption key from the root key. */ static const unsigned char *g_encryption_key_salt_format; /* MAC Key Salt format. This is used to derive the MAC key from the root key. */ diff --git a/src/include/keymgr/encrypt/security_client_logic_enums.h b/src/include/keymgr/encrypt/security_client_logic_enums.h new file mode 100644 index 000000000..755075f63 --- /dev/null +++ b/src/include/keymgr/encrypt/security_client_logic_enums.h @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * security_client_logic_enums.h + * + * IDENTIFICATION + * src/gausskernel/security/keymgr/include/encrypt/security_client_logic_enums.h + * + * ------------------------------------------------------------------------- + */ + +#ifndef CLIENT_LOGIC_ENUMS_H +#define CLIENT_LOGIC_ENUMS_H + +#include + +typedef enum class CmkKeyStore { + INVALID_KEYSTORE = -1, + LOCALKMS, + GS_KTOOL, +} CmkKeyStore; + +typedef enum class CmkAlgorithm { + INVALID_ALGORITHM, + RSA_2048, + AES_256_CBC, + SM2, + SM4, + AES_256_GCM +} CmkAlgorithm; + +typedef enum class EncryptionType { + INVALID_TYPE, + RANDOMIZED_TYPE, + DETERMINISTIC_TYPE +} EncryptionType; + + +typedef enum ColumnEncryptionAlgorithm { + INVALID_ALGORITHM, + AEAD_AES_256_CBC_HMAC_SHA256, + AEAD_AES_128_CBC_HMAC_SHA256, + SM4_SM3, + AES_256_GCM_ALGO, + AES_256_CTR_ALGO +} ColumnEncryptionAlgorithm; + +inline ColumnEncryptionAlgorithm get_cek_algorithm_from_string(const char *alg) +{ + if (alg == NULL || strlen(alg) == 0) { + return ColumnEncryptionAlgorithm::INVALID_ALGORITHM; + } + if (strcasecmp(alg, "AEAD_AES_256_CBC_HMAC_SHA256") == 0) { + return ColumnEncryptionAlgorithm::AEAD_AES_256_CBC_HMAC_SHA256; + } else if (strcasecmp(alg, "AEAD_AES_128_CBC_HMAC_SHA256") == 0) { + return ColumnEncryptionAlgorithm::AEAD_AES_128_CBC_HMAC_SHA256; + } else if (strcasecmp(alg, "SM4_SM3") == 0) { + return ColumnEncryptionAlgorithm::SM4_SM3; + } else if (strcasecmp(alg, "AES_256_GCM") == 0) { + return ColumnEncryptionAlgorithm::AES_256_GCM_ALGO; + } + return ColumnEncryptionAlgorithm::INVALID_ALGORITHM; +} + +inline CmkKeyStore get_key_store_from_string(const char *key_store) +{ + if (key_store == NULL || strlen(key_store) == 0) { + return CmkKeyStore::INVALID_KEYSTORE; + } + if (strcasecmp(key_store, "gs_ktool") == 0) { + return CmkKeyStore::GS_KTOOL; + } + if (strcasecmp(key_store, "localkms") == 0) { + return CmkKeyStore::LOCALKMS; + } + return CmkKeyStore::INVALID_KEYSTORE; +} + +inline CmkAlgorithm get_algorithm_from_string(const char *algorithm) +{ + if (algorithm == NULL || strlen(algorithm) == 0) { + return CmkAlgorithm::INVALID_ALGORITHM; + } + + if (strcasecmp(algorithm, "AES_256_CBC") == 0) { + return CmkAlgorithm::AES_256_CBC; + } else if (strcasecmp(algorithm, "SM4") == 0) { + return CmkAlgorithm::SM4; + } else if (strcasecmp(algorithm, "RSA_2048") == 0) { + return CmkAlgorithm::RSA_2048; + } else if (strcasecmp(algorithm, "SM2") == 0) { + return CmkAlgorithm::SM2; + } else if (strcasecmp(algorithm, "AES_256_GCM") == 0) { + return CmkAlgorithm::AES_256_GCM; + } + + return CmkAlgorithm::INVALID_ALGORITHM; +} + +inline EncryptionType get_algorithm_type_from_string(const char *algorithm_type) +{ + if (algorithm_type == NULL || strlen(algorithm_type) == 0) { + return EncryptionType::INVALID_TYPE; + } + if (strcasecmp(algorithm_type, "DETERMINISTIC") == 0) { + return EncryptionType::DETERMINISTIC_TYPE; + } else if (strcasecmp(algorithm_type, "RANDOMIZED") == 0) { + return EncryptionType::RANDOMIZED_TYPE; + } + + return EncryptionType::INVALID_TYPE; +} +#endif /* CLIENT_LOGIC_ENUMS_H */ diff --git a/src/common/interfaces/libpq/client_logic_hooks/encryption_hooks/encrypt_decrypt.h b/src/include/keymgr/encrypt/security_encrypt_decrypt.h similarity index 76% rename from src/common/interfaces/libpq/client_logic_hooks/encryption_hooks/encrypt_decrypt.h rename to src/include/keymgr/encrypt/security_encrypt_decrypt.h index 5f7c5fa2d..6af4c8784 100644 --- a/src/common/interfaces/libpq/client_logic_hooks/encryption_hooks/encrypt_decrypt.h +++ b/src/include/keymgr/encrypt/security_encrypt_decrypt.h @@ -13,21 +13,34 @@ * See the Mulan PSL v2 for more details. * ------------------------------------------------------------------------- * - * encrypt_decrypt.h + * security_encrypt_decrypt.h * * IDENTIFICATION - * src\common\interfaces\libpq\client_logic_hooks\encryption_hooks\encrypt_decrypt.h + * src/gausskernel/security/keymgr/include/encrypt/security_encrypt_decrypt.h * * ------------------------------------------------------------------------- */ #ifndef HAVE_AES256_CBC_ENCRYPTION_INCLUDE #define HAVE_AES256_CBC_ENCRYPTION_INCLUDE -#include "client_logic/client_logic_enums.h" -#include "aead_aes_hamc_enc_key.h" -#include "cipher.h" +#include "keymgr/encrypt/security_client_logic_enums.h" +#include "keymgr/encrypt/security_aead_aes_hamc_enc_key.h" -/* +/* Params of encrypt and decrypt */ +typedef struct EncParam { + unsigned char *plaintext; + int plaintext_len; + unsigned char *ciphertext; + int ciphertext_len; + const unsigned char *key; + int key_len; + unsigned char *iv; + int iv_len; + unsigned char *tag; + int tag_len; +} EncParam; + +/* * returns the recommended size for allocation for encrypt_data's result buffer * returns value <= 0 when provided with invalid input */ @@ -46,16 +59,16 @@ int get_cipher_text_size(int plainTextSize); * returns the ciphertext length * return value == 0 on error */ -int encrypt_data(const unsigned char *plain_text, int plain_text_length, AeadAesHamcEncKey &column_encryption_key, +int encrypt_data(unsigned char *plain_text, int plain_text_length, AeadAesHamcEncKey &column_encryption_key, EncryptionType encryption_type, unsigned char *result, ColumnEncryptionAlgorithm column_encryption_algorithm); /* - * Decryption steps + * Decryption steps * 1. Validate version byte * 2. Validate Authentication tag * 3. Decrypt the message */ -int decrypt_data(const unsigned char *cipher_text, int cipher_text_length, +int decrypt_data(unsigned char *cipher_text, int cipher_text_length, AeadAesHamcEncKey &column_encryptionKey, unsigned char *decryptedtext, ColumnEncryptionAlgorithm column_encryption_algorithm); diff --git a/src/common/interfaces/libpq/client_logic_hooks/encryption_hooks/sm2_enc_key.h b/src/include/keymgr/encrypt/security_sm2_enc_key.h similarity index 88% rename from src/common/interfaces/libpq/client_logic_hooks/encryption_hooks/sm2_enc_key.h rename to src/include/keymgr/encrypt/security_sm2_enc_key.h index 081fd98c0..371ff2539 100644 --- a/src/common/interfaces/libpq/client_logic_hooks/encryption_hooks/sm2_enc_key.h +++ b/src/include/keymgr/encrypt/security_sm2_enc_key.h @@ -13,10 +13,10 @@ * See the Mulan PSL v2 for more details. * ------------------------------------------------------------------------- * - * sm2_enc_key.h + * security_sm2_enc_key.h * * IDENTIFICATION - * src\common\interfaces\libpq\client_logic_hooks\encryption_hooks\sm2_enc_key.h + * src/gausskernel/security/keymgr/include/ecnrypt/security_sm2_enc_key.h * * ------------------------------------------------------------------------- */ @@ -24,7 +24,7 @@ #ifndef SM2_ENC_KEY #define SM2_ENC_KEY -#include "../cmk_entity_manager_hooks/cmkem_comm.h" +#include "keymgr/localkms/security_cmkem_comm.h" typedef struct Sm2KeyPair { CmkemUStr *pub_key; diff --git a/src/include/keymgr/his/security_his.h b/src/include/keymgr/his/security_his.h new file mode 100644 index 000000000..a6bda32af --- /dev/null +++ b/src/include/keymgr/his/security_his.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * security_his.h + * + * IDENTIFICATION + * src/gausskernel/security/keymgr/include/his/security_his.h + * + * ------------------------------------------------------------------------- + */ + +#ifndef __HIS_H_ +#define __HIS_H_ + +#include "keymgr/security_key_mgr.h" +#include "keymgr/his/security_his_iam.h" +#include "keymgr/his/security_his_kms.h" + +typedef struct { + KeyMgr kmgr; + + HisIamMgr *iammgr; + HisKmsMgr *kmsmgr; + + int setarg; +} HisMgr; + +HisMgr *his_mgr_new(KmErr *err); +void his_mgr_free(HisMgr *his); +void his_mgr_set_arg(HisMgr *his, const char *key, const char *value); + +#define his_mgr_get_err(his) km_err_get_msg(((HisMgr *)(his))->kmgr.err) +#define his_mgr_reset_err(his) km_err_reset((his)->kmgr.err) + +extern KeyMethod his_kms; + +#endif \ No newline at end of file diff --git a/src/include/keymgr/his/security_his_iam.h b/src/include/keymgr/his/security_his_iam.h new file mode 100644 index 000000000..8025314f0 --- /dev/null +++ b/src/include/keymgr/his/security_his_iam.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * security_his_iam.h + * + * IDENTIFICATION + * src/gausskernel/security/keymgr/include/include/his/security_his_iam.h + * + * ------------------------------------------------------------------------- + */ + +#ifndef __HIS_IAM_H_ +#define __HIS_IAM_H_ + +#include "keymgr/comm/security_http.h" + +typedef struct { + char *url; + char *account; + char *secret; + char *appid; + char *enterprise; + + char *cacert; + HttpMgr *httpmgr; + + char *reqbody; + char *token; + + KmErr *err; +} HisIamMgr; + +HisIamMgr *his_iam_new(KmErr *errbuf); +void his_iam_free(HisIamMgr *iam); + +void his_iam_set_arg(HisIamMgr *iam, const char *key, const char *value); +char *his_iam_get_token(HisIamMgr *iam); +char *his_iam_refresh_token(HisIamMgr *iam); + +#endif \ No newline at end of file diff --git a/src/include/keymgr/his/security_his_kms.h b/src/include/keymgr/his/security_his_kms.h new file mode 100644 index 000000000..8a4da3bba --- /dev/null +++ b/src/include/keymgr/his/security_his_kms.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * security_his_kms.h + * + * IDENTIFICATION + * src/gausskernel/security/keymgr/include/his/security_his_kms.h + * + * ------------------------------------------------------------------------- + */ + +#ifndef __HIS_KMS_H_ +#define __HIS_KMS_H_ + +#include "keymgr/comm/security_http.h" +#include "keymgr/comm/security_utils.h" +#include "keymgr/his/security_his_iam.h" + +#define KMS_URL_BUF_SZ 256 + +typedef struct { + char *appid; + /* default: his-op */ + char *domain; + char *kmsurl; + + HisIamMgr *iam; + + char url[KMS_URL_BUF_SZ]; + char *reqbody; + + char *cacert; + HttpMgr *httpmgr; + + KmErr *err; +} HisKmsMgr; + +HisKmsMgr *his_kms_new(HisIamMgr *iam, KmErr *errbuf); +void his_kms_free(HisKmsMgr *kms); +void his_kms_set_arg(HisKmsMgr *kms, const char *key, const char *value); + +char *his_kms_mk_select(HisKmsMgr *kms, const char *keypath); +KmUnStr his_kms_mk_encrypt(HisKmsMgr *kms, const char *keypath, KmUnStr plain); +KmUnStr his_kms_mk_decrypt(HisKmsMgr *kms, const char *keypath, KmUnStr cipher); + +KmUnStr his_kms_dk_create(HisKmsMgr *kms, const char *keypath, KmUnStr *cipher); +#endif \ No newline at end of file diff --git a/src/include/keymgr/hwc/security_hwc.h b/src/include/keymgr/hwc/security_hwc.h new file mode 100644 index 000000000..f391cc580 --- /dev/null +++ b/src/include/keymgr/hwc/security_hwc.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * security_hwc.h + * + * IDENTIFICATION + * src/gausskernel/security/keymgr/include/hwc/security_hwc.h + * + * ------------------------------------------------------------------------- + */ + +#ifndef __HWC_H_ +#define __HWC_H_ + +#include "keymgr/security_key_mgr.h" +#include "keymgr/hwc/security_hwc_iam.h" +#include "keymgr/hwc/security_hwc_kms.h" + +typedef struct { + KeyMgr kmgr; + + HwcIamMgr *iammgr; + HwcKmsMgr *kmsmgr; + + int setarg; +} HwcMgr; + +HwcMgr *hwc_mgr_new(KmErr *err); +void hwc_mgr_free(HwcMgr *hwc); +void hwc_mgr_set_arg(HwcMgr *hwc, const char *key, const char *value); + +extern KeyMethod huawei_kms; +extern KeyMethod hcs_kms; + +#endif \ No newline at end of file diff --git a/src/include/keymgr/hwc/security_hwc_iam.h b/src/include/keymgr/hwc/security_hwc_iam.h new file mode 100644 index 000000000..be0b1f480 --- /dev/null +++ b/src/include/keymgr/hwc/security_hwc_iam.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * security_hwc_iam.h + * + * IDENTIFICATION + * src/gausskernel/security/keymgr/include/hwc/security_hwc_iam.h + * + * ------------------------------------------------------------------------- + */ + +#ifndef __HWC_IAM_H_ +#define __HWC_IAM_H_ + +#include "keymgr/comm/security_http.h" + +typedef struct { + char *url; + char *username; + char *passwd; + char *domain; + char *project; + + HttpMgr *httpmgr; + char *cacert; + + char *reqbody; + char *token; + + KmErr *err; +} HwcIamMgr; + +HwcIamMgr *hwc_iam_new(KmErr *errbuf); +void hwc_iam_free(HwcIamMgr *iam); +void hwc_iam_set_arg(HwcIamMgr *iam, const char *key, const char *value); + +char *hwc_iam_get_token(HwcIamMgr *iam); +char *hwc_iam_refresh_token(HwcIamMgr *iam); + +#endif \ No newline at end of file diff --git a/src/include/keymgr/hwc/security_hwc_kms.h b/src/include/keymgr/hwc/security_hwc_kms.h new file mode 100644 index 000000000..e13470440 --- /dev/null +++ b/src/include/keymgr/hwc/security_hwc_kms.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * security_hwc_kms.h + * + * IDENTIFICATION + * src/gausskernel/security/keymgr/include/hwc/security_hwc_kms.h + * + * ------------------------------------------------------------------------- + */ + +#ifndef __HWC_KMS_H_ +#define __HWC_KMS_H_ + +#include "keymgr/hwc/security_hwc_iam.h" +#include "keymgr/comm/security_http.h" +#include "keymgr/comm/security_utils.h" + +#define HWC_KMS_URL_SZ 256 +#define HWC_PROJ_ID_HDR_SZ 128 + +typedef struct { + bool aksk; + HwcIamMgr *iam; + + char url[HWC_KMS_URL_SZ]; + char projid[HWC_PROJ_ID_HDR_SZ]; /* format: 'x-project-id: $kmsProjectId' */ + char *reqbody; + HttpMgr *httpmgr; + char *cacert; + + KmErr *err; +} HwcKmsMgr; + +HwcKmsMgr *hwc_kms_new(HwcIamMgr *iam, KmErr *errbuf); +void hwc_kms_free(HwcKmsMgr *kms); +void hwc_kms_set_arg(HwcKmsMgr *kms, const char *key, const char *value); + +char *hwc_kms_mk_select(HwcKmsMgr *kms, const char *keypath); +KmUnStr hwc_kms_mk_encrypt(HwcKmsMgr *kms, const char *keypath, KmUnStr plain); +KmUnStr hwc_kms_mk_decrypt(HwcKmsMgr *kms, const char *keypath, KmUnStr cipher); +KmUnStr hwc_kms_dk_create(HwcKmsMgr *kms, const char *keypath, KmUnStr *cipher); +#endif \ No newline at end of file diff --git a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/register_gs_ktool.h b/src/include/keymgr/ktool/security_gs_ktool.h similarity index 72% rename from src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/register_gs_ktool.h rename to src/include/keymgr/ktool/security_gs_ktool.h index d3c76b9cb..bfe82c259 100644 --- a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/register_gs_ktool.h +++ b/src/include/keymgr/ktool/security_gs_ktool.h @@ -13,7 +13,7 @@ * See the Mulan PSL v2 for more details. * ------------------------------------------------------------------------- * - * register_gs_ktool.h + * security_gs_ktool.h * gs_ktool is an independent key management tool provided by GaussDB Kernel, can generate and store symmetric * key with [16, 112] bytes. * when CREATE CMKO, if KEY_STROE = gs_ktool, then: @@ -22,15 +22,18 @@ * available for AES_256 algorithm * if you register gs_ktool, you should be sure your system has installed gs_ktool, and the environment variables * and the configuration files are available. - * + * * IDENTIFICATION - * src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/register_gs_ktool.h + * src/gausskernel/security/keymgr/include/ktool/security_gs_ktool.h * * ------------------------------------------------------------------------- */ -#ifndef REGISTER_GS_KTOOL_H -#define REGISTER_GS_KTOOL_H + +#ifndef __KM_GS_KTOOL_H_ +#define __KM_GS_KTOOL_H_ + +#include "keymgr/security_key_mgr.h" const int MAX_CMK_CACHE_NODE_CNT = 100; const int DEFAULT_CMK_CACHE_LEN = 32; @@ -46,12 +49,12 @@ typedef struct CmkCacheList { CmkCacheNode *first_cmk_node; } CmkCacheList; -extern CmkCacheList *init_cmk_cache_list(); -extern void push_cmk_to_cache(CmkCacheList *cmk_cache_list, unsigned int cmk_id, const unsigned char *cmk_plian); -extern bool get_cmk_from_cache(CmkCacheList *cmk_cache_list, unsigned int cmk_id, unsigned char *cmk_plain); -extern void free_cmk_cache_list(CmkCacheList *cmk_cahce_list); +typedef struct { + KeyMgr kmgr; + + CmkCacheList *cache; +} GsKtoolMgr; -extern int reg_cmke_manager_gs_ktool_main(); -extern void exit_cmke_manager_gs_ktool(); +extern KeyMethod gs_ktool; -#endif /* REGISTER_GS_KTOOL_H */ +#endif diff --git a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/cmkem_comm.h b/src/include/keymgr/localkms/security_cmkem_comm.h similarity index 73% rename from src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/cmkem_comm.h rename to src/include/keymgr/localkms/security_cmkem_comm.h index 633d5f60a..c95b6ea62 100644 --- a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/cmkem_comm.h +++ b/src/include/keymgr/localkms/security_cmkem_comm.h @@ -13,14 +13,14 @@ * See the Mulan PSL v2 for more details. * ------------------------------------------------------------------------- * - * cmkem_comm.h + * security_cmkem_comm.h * some common functions, include: * 1. error code and error process * 2. string process * 3. format and conversion - * + * * IDENTIFICATION - * src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/cmkem_comm.h + * src/gausskernel/security/keymgr/include/localkms/security_cmkem_comm.h * * ------------------------------------------------------------------------- */ @@ -133,9 +133,6 @@ typedef struct CmkIdentity { const char* cmk_store; const char *cmk_id_str; const char *cmk_algo; - - unsigned int cmk_id_num; - size_t client_cache_id; } CmkIdentity; #define cmkem_errmsg(format_str, ...) \ @@ -162,51 +159,17 @@ typedef struct CmkIdentity { } \ } while (0) -#define cmkem_list_free(ptr) \ - do { \ - if ((ptr) != NULL) { \ - free_cmkem_list((ptr)); \ - (ptr) = NULL; \ - } \ - } while (0) - /* error process */ extern void write_cmkem_errmsg(const char *errmsg); -extern void print_cmkem_errmsg_buf(); -extern void dump_cmkem_errmsg_buf(char *dest_buf, size_t dest_buf_size); extern const char *get_cmkem_errmsg(CmkemErrCode err_code); extern void erase_data(void *data, size_t data_len); /* advanced string */ -extern CmkemStr *malloc_cmkem_str(size_t str_buf_len); extern void free_cmkem_str(CmkemStr *cmkem_str_ptr); extern CmkemUStr *malloc_cmkem_ustr(size_t key_buf_len); extern void free_cmkem_ustr(CmkemUStr *cmkem_ustr); extern void free_cmkem_ustr_with_erase(CmkemUStr *cmkem_ustr); -extern CmkemStrList *malloc_cmkem_list(void); -extern void free_cmkem_list(CmkemStrList *list); -extern void free_cmkem_list_with_skip(CmkemStrList *list, int list_pos); - -extern size_t cmkem_list_len(CmkemStrList *list); -extern void cmkem_list_size(CmkemStrList *list, size_t *list_size); -extern char *cmkem_list_val(CmkemStrList *list, int list_pos); -extern int cmkem_list_pos(CmkemStrList *list, const char *str_val); -extern void cmkem_list_print(CmkemStrList *list); -extern void cmkem_list_append(CmkemStrList *list, const char *str_val); -extern void cmkem_list_insert(CmkemStrList *list, const char *str_val, int list_pos); -extern void cmkem_list_pop(CmkemStrList *list); -extern void cmkem_list_del(CmkemStrList *list, int list_pos); -extern void cmkem_list_cat(CmkemStrList *base_list, CmkemStrList *extra_list); -extern char *cmkem_list_merge(CmkemStrList *list); -extern CmkemStrList *cmkem_split(const char *str, char split_char); - -extern bool is_str_empty(const char *ptr); -extern CmkemStr *conv_str_to_cmkem_str(const char *str); -extern CmkemUStr *conv_str_to_cmkem_ustr(const unsigned char *ustr, size_t ustr_len); -extern void push_char(CmkemStr *cmkem_str_ptr, char chr); - -extern void itoa(int num, char *str_buf, size_t buf_len); -extern int hex_atoi(char hex); + extern CmkemStr *ustr_to_hex(CmkemUStr *ustr); extern CmkemUStr *hex_to_ustr(CmkemStr *hex); diff --git a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/cmkem_comm_algorithm.h b/src/include/keymgr/localkms/security_cmkem_comm_algorithm.h similarity index 78% rename from src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/cmkem_comm_algorithm.h rename to src/include/keymgr/localkms/security_cmkem_comm_algorithm.h index 8496f0f0a..a93a9ca51 100644 --- a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/cmkem_comm_algorithm.h +++ b/src/include/keymgr/localkms/security_cmkem_comm_algorithm.h @@ -13,12 +13,12 @@ * See the Mulan PSL v2 for more details. * ------------------------------------------------------------------------- * - * cmkem_comm_algorithm.h + * security_cmkem_comm_algorithm.h * some general encryption and decryption function. * you can use them to encrypt and decrypt your data, CEK entity and CMK entity. - * + * * IDENTIFICATION - * src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/cmkem_comm_algorithm.h + * src/gausskernel/security/keymgr/include/localkms/security_cmkem_comm_algorithm.h * * ------------------------------------------------------------------------- */ @@ -27,23 +27,25 @@ #define CMEKM_COMM_ALGORITHM #include -#include "cmkem_comm.h" +#include "keymgr/localkms/security_cmkem_comm.h" const int AES256_KEY_BUF_LEN = 257; const int AES256_KEY_LEN = 256; const int RSA2048_KEN_LEN = 2048; -const int RSA3072_KEN_LEN = 2048; +const int RSA3072_KEN_LEN = 3072; const int SHA256_HASH_LEN = 32; const int MAX_ASYMM_KEY_BUF_LEN = 3072; typedef enum { UNKNOWN_ALGO, - AES_256_CBC, - SM4, - RSA_2048, - RSA_3072, - SM2, + AT_AES_256_CBC, + AT_SM4_CBC, + AT_RSA_2048, + AT_RSA_3072, + AT_SM2, + AT_AES_256_GCM, + AT_AES_256_CTR, } AlgoType; typedef enum { @@ -53,8 +55,6 @@ typedef enum { extern size_t get_key_len_by_algo(AlgoType cmk_algo); extern AlgoType get_algo_by_str(const char *algo_str); -extern CmkemErrCode get_hex_join_hash_from_ustr(CmkemUStr *ustr, CmkemStr **hex_join_hash); -extern CmkemErrCode get_ustr_from_hex_join_hash(CmkemStr *hex_join_hash, CmkemUStr **ustr); extern CmkemErrCode encrypt_with_symm_algo(AlgoType algo, CmkemUStr *cek_plain, CmkemUStr *cmk_plain, CmkemUStr **cek_cipher); @@ -69,7 +69,4 @@ extern CmkemErrCode read_rsa_key_from_biobuf(BIO *key_biobuf, CmkemUStr **key); extern CmkemErrCode encrypt_with_rsa2048_pub_key(CmkemUStr *cek_plain, CmkemUStr *cmk_plain, CmkemUStr **cek_cipher); extern CmkemErrCode decrypt_with_rsa2048_priv_key(CmkemUStr *cek_cipher, CmkemUStr *cmk_plain, CmkemUStr **cek_plain); -extern CmkemErrCode join_sha256(CmkemUStr *data, CmkemUStr **data_join_sha256); -extern CmkemErrCode strip_sha256(CmkemUStr *data_join_sha256, CmkemUStr **data); - #endif /* CMEKM_COMM_ALGORITHM */ diff --git a/src/include/keymgr/localkms/security_file_enc.h b/src/include/keymgr/localkms/security_file_enc.h new file mode 100644 index 000000000..55638e881 --- /dev/null +++ b/src/include/keymgr/localkms/security_file_enc.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * security_file_enc.h + * + * IDENTIFICATION + * src/gausskernel/security/keymgr/include/localkms/security_file_enc.h + * + * ------------------------------------------------------------------------- + */ + +#ifndef _KM_FILE_ENC_H_ +#define _KM_FILE_ENC_H_ + +#include "keymgr/localkms/security_cmkem_comm_algorithm.h" + +typedef enum { + CREATE_KEY_FILE, + READ_KEY_FILE +} CheckKeyFileType; + +typedef enum { + PUB_KEY_FILE, + PRIV_KEY_FILE, + PUB_KEY_ENC_IV_FILE, + PRIV_KEY_ENC_IV_FILE, +} LocalkmsFileType; + +CmkemErrCode set_global_env_value(); +void get_file_path_from_cmk_id(const char *cmk_id_str, LocalkmsFileType file_type, char *file_path_buf, size_t buf_len); +CmkemErrCode check_file_exist(const char *real_file_path, CheckKeyFileType chk_type); +CmkemErrCode encrypt_and_write_key(const char *key_file_path, CmkemUStr *key_plain); +CmkemErrCode read_and_decrypt_cmk(const char *key_path, AsymmetricKeyType key_type, CmkemUStr **key_palin); + +#endif /* __KM_FILE_ENC_H__ */ diff --git a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/register_local_kms.h b/src/include/keymgr/localkms/security_localkms.h similarity index 81% rename from src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/register_local_kms.h rename to src/include/keymgr/localkms/security_localkms.h index 9eb0efe12..618ac5ef2 100644 --- a/src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/register_local_kms.h +++ b/src/include/keymgr/localkms/security_localkms.h @@ -13,23 +13,31 @@ * See the Mulan PSL v2 for more details. * ------------------------------------------------------------------------- * - * register_local_kms.h + * security_localkms.h * localkms is a lightweight key management component. * different from other key management tools and services, localkms automatically generates CMKE when CREATE CMKO, * it cannot manage key entities independently. * localkms use openssl to generate cmk, then encrypt cmk plain and store cmk cipher in file. * at the same time, we need to store/read the iv and salt that are used to derive a key to * encrypt/decrypt cmk plain. - * + * * IDENTIFICATION - * src/common/interfaces/libpq/client_logic_hooks/cmk_entity_manager_hooks/register_local_kms.h + * src/gausskernel/security/keymgr/include/localkms/security_localkms.h * * ------------------------------------------------------------------------- */ -#ifndef REGISTER_LOCAL_KMS_H -#define REGISTER_LOCAL_KMS_H +#ifndef __KM_LOCALKMS_H__ +#define __KM_LOCALKMS_H__ + +#include "keymgr/security_key_mgr.h" + +typedef struct { + KeyMgr kmgr; + + int getpath; +} LocalKmsMgr; -extern int reg_cmke_manager_local_kms_main(); +extern KeyMethod localkms; #endif /* REG_LOCALKMS_H */ diff --git a/src/include/keymgr/security_key_adpt.h b/src/include/keymgr/security_key_adpt.h new file mode 100644 index 000000000..b23cfae05 --- /dev/null +++ b/src/include/keymgr/security_key_adpt.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * security_key_adpt.h + * + * IDENTIFICATION + * src/gausskernel/security/keymgr/include/keymgr/security_key_adpt.cpp + * + * ------------------------------------------------------------------------- + */ + +#ifndef _KM_KEY_ADPT_H_ +#define _KM_KEY_ADPT_H_ + +#include "keymgr/security_key_mgr.h" + +#define MAX_KEY_MGR_NUM 20 + +typedef struct { + KmErr *err; + + KeyMethod *methods[MAX_KEY_MGR_NUM]; + KeyMgr *mgrs[MAX_KEY_MGR_NUM]; +} KeyAdpt; + +KeyAdpt *key_adpt_new(); +void key_adpt_free(KeyAdpt *kadpt); +void key_adpt_set_arg(KeyAdpt *kadpt, const char *type, const char *key, const char *value); +void key_adpt_set_args(KeyAdpt *kadpt, const char *type, const char *args); + +void key_adpt_mk_create(KeyAdpt *kadpt, KeyInfo info); +void key_adpt_mk_delete(KeyAdpt *kadpt, KeyInfo info); +char *key_adpt_mk_select(KeyAdpt *kadpt, KeyInfo info); +KmUnStr key_adpt_mk_encrypt(KeyAdpt *kadpt, KeyInfo info, KmUnStr plain); +KmUnStr key_adpt_mk_decrypt(KeyAdpt *kadpt, KeyInfo info, KmUnStr cipher); + +KmUnStr key_adpt_dk_create(KeyAdpt *kadpt, KeyInfo info, KmUnStr *cipher); + +bool key_adpt_catch_err(KeyAdpt *kadpt); +const char *key_adpt_get_err(KeyAdpt *kadpt); + +#endif diff --git a/src/include/keymgr/security_key_mgr.h b/src/include/keymgr/security_key_mgr.h new file mode 100644 index 000000000..37fa44471 --- /dev/null +++ b/src/include/keymgr/security_key_mgr.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * security_key_mgr.h + * + * IDENTIFICATION + * src/gausskernel/security/keymgr/include/keymgr/security_key_mgr.h + * + * ------------------------------------------------------------------------- + */ + +#ifndef _KM_KEY_MGR_H_ +#define _KM_KEY_MGR_H_ + +#include "keymgr/comm/security_error.h" +#include "keymgr/comm/security_utils.h" + +/* KeyInfo */ +typedef struct { + const char *type; + const char *id; + const char *algo; +} KeyInfo; + +/* KeyMgr */ +typedef struct { + KmErr *err; + + /* special space */ +} KeyMgr; + +/* KeyMgrMethod */ +typedef KeyMgr* (*KeyMgrNew)(KmErr *err); +typedef void (*KeyMgrFree)(KeyMgr *kmgr); +typedef void (*KeyMgrSetArg)(KeyMgr *kmgr, const char *key, const char *value); + +/* KeyMethod */ +typedef void (*MasterKeyCreate)(KeyMgr *kmgr, KeyInfo info); +typedef void (*MasterKeyDelete)(KeyMgr *kmgr, KeyInfo info); +typedef KmUnStr (*MasterKeyEncrypt)(KeyMgr *kmgr, KeyInfo info, KmUnStr plain); +typedef KmUnStr (*MasterKeyDecrypt)(KeyMgr *kmgr, KeyInfo info, KmUnStr cipher); +typedef char* (*MasterKeySelect)(KeyMgr *kmgr, KeyInfo info); + +typedef KmUnStr (*DataKeyCreate)(KeyMgr *kmgr, KeyInfo info, KmUnStr *cipher); + +typedef struct { + const char *type; + + /* key mgr method */ + KeyMgrNew kmgr_new; + KeyMgrFree kmgr_free; + KeyMgrSetArg kmgr_set_arg; + + /* key method */ + MasterKeyCreate mk_create; + MasterKeyDelete mk_delete; + MasterKeySelect mk_select; + MasterKeyEncrypt mk_enrypt; + MasterKeyDecrypt mk_decrypt; + + DataKeyCreate dk_create; +} KeyMethod; + +#endif + diff --git a/src/include/libpq/cl_state.h b/src/include/libpq/cl_state.h index a44ad035a..abad99fb6 100644 --- a/src/include/libpq/cl_state.h +++ b/src/include/libpq/cl_state.h @@ -110,6 +110,7 @@ class PGClientLogic { size_t called_functions_list_size; ProcList *proc_list; size_t client_cache_id; + void *m_key_adpt; // sql parser context clientlogic_parser_context m_parser_context; @@ -117,4 +118,6 @@ class PGClientLogic { private: const ICachedRec* get_cl_proc(const char* pname) const; }; + +#define CL_GET_KEY_ADPT(clientlogic) (KeyAdpt *)(void *)((clientlogic).m_key_adpt) #endif /* CL_STATE_H */ From 0c664d5a8be880ac5154b786e7ae0bd016bc77df Mon Sep 17 00:00:00 2001 From: zhang_xubo <2578876417@qq.com> Date: Tue, 25 Jul 2023 21:30:52 +0800 Subject: [PATCH 053/304] package add tool pagehack and xlogdump --- build/script/aarch64_opengauss_list | 2 ++ build/script/utils/make_compile.sh | 15 +++++++++++++++ build/script/x86_64_opengauss_list | 2 ++ 3 files changed, 19 insertions(+) diff --git a/build/script/aarch64_opengauss_list b/build/script/aarch64_opengauss_list index fd36832e0..6d543cee0 100644 --- a/build/script/aarch64_opengauss_list +++ b/build/script/aarch64_opengauss_list @@ -45,6 +45,8 @@ ./bin/server.key.cipher ./bin/server.key.rand ./bin/gs_plan_simulator.sh +./bin/pg_xlogdump +./bin/pagehack ./etc/kerberos/kadm5.acl ./etc/kerberos/kdc.conf ./etc/kerberos/krb5.conf diff --git a/build/script/utils/make_compile.sh b/build/script/utils/make_compile.sh index 6056eac1e..375958fbf 100644 --- a/build/script/utils/make_compile.sh +++ b/build/script/utils/make_compile.sh @@ -28,6 +28,9 @@ function read_gaussdb_version() PG_REG_TEST_ROOT="${ROOT_DIR}" ROACH_DIR="${ROOT_DIR}/distribute/bin/roach" MPPDB_DECODING_DIR="${ROOT_DIR}/contrib/mppdb_decoding" +XLOG_DUMP_DIR="${ROOT_DIR}/contrib/pg_xlogdump" +PAGE_HACK_DIR="${ROOT_DIR}/contrib/pagehack" + ################################### # get version number from globals.cpp @@ -259,6 +262,18 @@ function install_gaussdb() fi fi + cd "$XLOG_DUMP_DIR" + make clean >> "$LOG_FILE" 2>&1 + make -sj >> "$LOG_FILE" 2>&1 + make install -sj >> "$LOG_FILE" 2>&1 + echo "End make install xlog_dump" >> "$LOG_FILE" 2>&1 + + cd "$PAGE_HACK_DIR" + make clean >> "$LOG_FILE" 2>&1 + make -sj >> "$LOG_FILE" 2>&1 + make install -sj >> "$LOG_FILE" 2>&1 + echo "End make install pagehack" >> "$LOG_FILE" 2>&1 + chmod 444 ${BUILD_DIR}/bin/cluster_guc.conf dos2unix ${BUILD_DIR}/bin/cluster_guc.conf > /dev/null 2>&1 get_kernel_commitid diff --git a/build/script/x86_64_opengauss_list b/build/script/x86_64_opengauss_list index 5e2f16616..500129a89 100644 --- a/build/script/x86_64_opengauss_list +++ b/build/script/x86_64_opengauss_list @@ -45,6 +45,8 @@ ./bin/server.key.cipher ./bin/server.key.rand ./bin/gs_plan_simulator.sh +./bin/pg_xlogdump +./bin/pagehack ./etc/kerberos/kadm5.acl ./etc/kerberos/kdc.conf ./etc/kerberos/krb5.conf From 28d1e9c6643c46d7ded420e5f64c0bdf883e13d7 Mon Sep 17 00:00:00 2001 From: duzhuolin Date: Wed, 26 Jul 2023 00:01:38 +0800 Subject: [PATCH 054/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dprepjointree=20?= =?UTF-8?q?=E8=8A=82=E7=82=B9=E6=9C=AA=E9=87=8A=E6=94=BE=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/optimizer/prep/prepjointree.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/gausskernel/optimizer/prep/prepjointree.cpp b/src/gausskernel/optimizer/prep/prepjointree.cpp index cbcfac1c2..18838537d 100755 --- a/src/gausskernel/optimizer/prep/prepjointree.cpp +++ b/src/gausskernel/optimizer/prep/prepjointree.cpp @@ -3858,9 +3858,13 @@ static bool is_safe_pull_up_sublink_having(PlannerInfo* root) subQuery = castNode(Query, sublink->subselect); level_up_varnos = pull_varnos((Node*)subQuery->jointree, 1, true); if (!bms_is_empty(level_up_varnos)) { + bms_free(level_up_varnos); + list_free_ext(sublinkList); return false; } } - + + bms_free(level_up_varnos); + list_free_ext(sublinkList); return true; } \ No newline at end of file From 73ab56ff307936c805e6cab045fb40a84f497b52 Mon Sep 17 00:00:00 2001 From: duzhuolin Date: Wed, 26 Jul 2023 00:07:07 +0800 Subject: [PATCH 055/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E8=8A=82=E7=82=B9?= =?UTF-8?q?=E6=9C=AA=E9=87=8A=E6=94=BE=E5=AF=BC=E8=87=B4mem=5FLeak?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/storage/access/transam/xlog.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/gausskernel/storage/access/transam/xlog.cpp b/src/gausskernel/storage/access/transam/xlog.cpp index 015d48803..0c444f611 100755 --- a/src/gausskernel/storage/access/transam/xlog.cpp +++ b/src/gausskernel/storage/access/transam/xlog.cpp @@ -7289,6 +7289,7 @@ static void DropAllRecoverySlotForPitr() List *all_archive_slots = NIL; all_archive_slots = GetAllRecoverySlotsName(); if (all_archive_slots == NIL || all_archive_slots->length == 0) { + list_free_deep(all_archive_slots); return; } From 85f51b73676d308a79ef81e886e2841aa2bed580 Mon Sep 17 00:00:00 2001 From: duzhuolin Date: Wed, 26 Jul 2023 00:17:21 +0800 Subject: [PATCH 056/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E8=8A=82=E7=82=B9?= =?UTF-8?q?=E6=9C=AA=E9=87=8A=E6=94=BE=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/storage/access/transam/multi_redo_settings.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/gausskernel/storage/access/transam/multi_redo_settings.cpp b/src/gausskernel/storage/access/transam/multi_redo_settings.cpp index 6ed92595e..e26dba43a 100644 --- a/src/gausskernel/storage/access/transam/multi_redo_settings.cpp +++ b/src/gausskernel/storage/access/transam/multi_redo_settings.cpp @@ -148,6 +148,7 @@ void ParseBindCpuInfo(RedoCpuBindControl *control) int bindNum = 0; if (strncmp("nobind", ptoken, strlen("nobind")) == 0) { control->bindType = REDO_NO_CPU_BIND; + pfree_ext(ptoken); return; } else if (strncmp("cpubind", ptoken, strlen("cpubind")) == 0) { control->bindType = REDO_CPU_BIND; @@ -169,6 +170,7 @@ void ParseBindCpuInfo(RedoCpuBindControl *control) "2. The process has been bind to other CPUs and there is no intersection," "use taskset -pc to check process CPU bind info.\n"))); } + pfree_ext(ptoken); } bool CpuCanConfiged(RedoCpuBindControl *contrl, int cpuid, int numaid) @@ -193,6 +195,7 @@ void ConfigBindCpuInfo(RedoCpuBindControl *contrl) ThreadPoolControler::GetActiveCpu(sysNumaCpuIdList, &sysNumaCpuIdNum); if (sysNumaCpuIdNum == 0) { + pfree_ext(sysNumaCpuIdList); return; } From f36604722eb5a272483564ac945ec723dae8ddca Mon Sep 17 00:00:00 2001 From: duzhuolin Date: Wed, 26 Jul 2023 00:27:28 +0800 Subject: [PATCH 057/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dsubselect=E8=8A=82?= =?UTF-8?q?=E7=82=B9=E6=9C=AA=E9=87=8A=E6=94=BE=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/optimizer/plan/subselect.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/gausskernel/optimizer/plan/subselect.cpp b/src/gausskernel/optimizer/plan/subselect.cpp index daae5c0fd..8c69659ae 100644 --- a/src/gausskernel/optimizer/plan/subselect.cpp +++ b/src/gausskernel/optimizer/plan/subselect.cpp @@ -6685,9 +6685,11 @@ static bool safe_pullup_uncorrelated_sublink_where(Node* inout_quals, Query* sub level_up_varnos = pull_varnos((Node*)subQuery->jointree, 1, true); if (!bms_is_empty(level_up_varnos) && bms_is_subset(level_up_varnos, *available_rels)) { + bms_free(level_up_varnos); return false; } - + + bms_free(level_up_varnos); if (!IsA(inout_quals, OpExpr)) { return false; } @@ -6721,9 +6723,11 @@ bool safe_pullup_op_expr_sublink(OpExpr* expr) // sublink should be only one sublinkList = pull_sublink((Node*)expr, 0, false, false); if (sublinkList == NULL || list_length(sublinkList) != 1) { + list_free_ext(sublinkList); return false; } - + + list_free_ext(sublinkList); return true; } From 40080f4a2404fa7b929cea304a0a23e60b6501ed Mon Sep 17 00:00:00 2001 From: duzhuolin Date: Wed, 26 Jul 2023 08:33:44 +0800 Subject: [PATCH 058/304] =?UTF-8?q?=E6=9C=AA=E9=87=8A=E6=94=BE=E8=8A=82?= =?UTF-8?q?=E7=82=B9=E5=AF=BC=E8=87=B4mem=5FLeak=E9=97=AE=E9=A2=98?= =?UTF-8?q?=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/optimizer/commands/copy.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/gausskernel/optimizer/commands/copy.cpp b/src/gausskernel/optimizer/commands/copy.cpp index 524954c5f..e64d5de5a 100644 --- a/src/gausskernel/optimizer/commands/copy.cpp +++ b/src/gausskernel/optimizer/commands/copy.cpp @@ -3547,6 +3547,7 @@ void BulkloadErrorCallback(void* arg) cstate->cur_lineno, cstate->cur_attname, lineval); + pfree_ext(lineval); } else { /* error is relevant to a particular line */ if (cstate->line_buf_converted || !cstate->need_transcoding) { From b7ffc7b7364ce856a9cac4a79ac741d10ba43030 Mon Sep 17 00:00:00 2001 From: ytwx1993 Date: Wed, 26 Jul 2023 09:28:39 +0800 Subject: [PATCH 059/304] =?UTF-8?q?=E5=90=8C=E6=AD=A53818?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/pgxc_single/pool/execRemote.cpp | 2 +- src/gausskernel/optimizer/commands/vacuumlazy.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/backend/pgxc_single/pool/execRemote.cpp b/src/common/backend/pgxc_single/pool/execRemote.cpp index cb6d6577f..00f7488f7 100755 --- a/src/common/backend/pgxc_single/pool/execRemote.cpp +++ b/src/common/backend/pgxc_single/pool/execRemote.cpp @@ -9017,7 +9017,7 @@ bool CheckPrepared(RemoteQuery* rq, Oid nodeoid) { Assert(false); DISTRIBUTED_FEATURE_NOT_SUPPORTED(); - return NULL; + return false; } void FindExecNodesInPBE(RemoteQueryState* planstate, ExecNodes* exec_nodes, RemoteQueryExecType exec_type) diff --git a/src/gausskernel/optimizer/commands/vacuumlazy.cpp b/src/gausskernel/optimizer/commands/vacuumlazy.cpp index 263e7debe..6593883fa 100644 --- a/src/gausskernel/optimizer/commands/vacuumlazy.cpp +++ b/src/gausskernel/optimizer/commands/vacuumlazy.cpp @@ -2115,7 +2115,7 @@ count_nondeletable_pages(Relation onerel, LVRelStats *vacrelstats) while (blkno > vacrelstats->nonempty_pages) { Buffer buf; Page page; - bool hastup = NULL; + bool hastup = false; /* * Check if another process requests a lock on our relation. We are From fac366983184143e058bea7b5cd896920b8f8361 Mon Sep 17 00:00:00 2001 From: ytwx1993 Date: Wed, 26 Jul 2023 09:50:44 +0800 Subject: [PATCH 060/304] =?UTF-8?q?contrib=E7=9B=AE=E5=BD=95=E4=B8=8B?= =?UTF-8?q?=E5=87=A0=E4=B8=AA=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- contrib/gauss_connector/gc_fdw.cpp | 4 ++-- contrib/pg_stat_statements/pg_stat_statements.cpp | 2 +- contrib/tablefunc/tablefunc.cpp | 4 ++-- contrib/tcn/tcn.cpp | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/contrib/gauss_connector/gc_fdw.cpp b/contrib/gauss_connector/gc_fdw.cpp index 15bcd2d80..567bc261f 100644 --- a/contrib/gauss_connector/gc_fdw.cpp +++ b/contrib/gauss_connector/gc_fdw.cpp @@ -163,8 +163,8 @@ static ForeignScan *gcGetForeignPlan(PlannerInfo *root, Oid foreigntableid, ForeignPath *best_path, List *tlist, - List *scan_clauses/*, - Plan *outer_plan*/); + List *scan_clauses, + Plan *outer_plan); static void gcBeginForeignScan(ForeignScanState* node, int eflags); static TupleTableSlot* gcIterateForeignScan(ForeignScanState* node); static VectorBatch* gcIterateVecForeignScan(VecForeignScanState* node); diff --git a/contrib/pg_stat_statements/pg_stat_statements.cpp b/contrib/pg_stat_statements/pg_stat_statements.cpp index 9ce5d9ca4..b91661ddc 100644 --- a/contrib/pg_stat_statements/pg_stat_statements.cpp +++ b/contrib/pg_stat_statements/pg_stat_statements.cpp @@ -135,7 +135,7 @@ typedef struct pgssEntry { * Global shared state */ typedef struct pgssSharedState { - LWLockId lock; /* protects hashtable search/modification */ + LWLock* lock; /* protects hashtable search/modification */ int query_size; /* max query length in bytes */ double cur_median_usage; /* current median usage in hashtable */ } pgssSharedState; diff --git a/contrib/tablefunc/tablefunc.cpp b/contrib/tablefunc/tablefunc.cpp index a41c7b923..3cea76986 100644 --- a/contrib/tablefunc/tablefunc.cpp +++ b/contrib/tablefunc/tablefunc.cpp @@ -1435,9 +1435,9 @@ static bool compatCrosstabTupleDescs(TupleDesc ret_tupdesc, TupleDesc sql_tupdes * attribute [2] of the sql tuple should match attributes [1] to [natts] * of the return tuple */ - sql_attr = sql_tupdesc->attrs[2]; + sql_attr = &sql_tupdesc->attrs[2]; for (i = 1; i < ret_tupdesc->natts; i++) { - ret_attr = ret_tupdesc->attrs[i]; + ret_attr = &ret_tupdesc->attrs[i]; if (ret_attr->atttypid != sql_attr->atttypid) return false; diff --git a/contrib/tcn/tcn.cpp b/contrib/tcn/tcn.cpp index 33c7a684a..2e9c73f79 100644 --- a/contrib/tcn/tcn.cpp +++ b/contrib/tcn/tcn.cpp @@ -152,7 +152,7 @@ Datum triggered_change_notification(PG_FUNCTION_ARGS) int colno = index->indkey.values[i]; appendStringInfoCharMacro(payload, ','); - strcpy_quoted(payload, NameStr((tupdesc->attrs[colno - 1])->attname), '"'); + strcpy_quoted(payload, NameStr((tupdesc->attrs[colno - 1]).attname), '"'); appendStringInfoCharMacro(payload, '='); strcpy_quoted(payload, SPI_getvalue(trigtuple, tupdesc, colno), '\''); } From a534c5e5150693a57b6035aae4a041d95de089e0 Mon Sep 17 00:00:00 2001 From: zhaojun Date: Wed, 26 Jul 2023 10:10:58 +0800 Subject: [PATCH 061/304] support auto startup dolphin-proto if needed --- src/gausskernel/process/postmaster/postmaster.cpp | 3 +++ src/gausskernel/process/tcop/postgres.cpp | 13 +++++++++++++ 2 files changed, 16 insertions(+) diff --git a/src/gausskernel/process/postmaster/postmaster.cpp b/src/gausskernel/process/postmaster/postmaster.cpp index c7e1ec2e4..5fab30e12 100644 --- a/src/gausskernel/process/postmaster/postmaster.cpp +++ b/src/gausskernel/process/postmaster/postmaster.cpp @@ -551,6 +551,7 @@ extern int ClientConnInitilize(Port* port); extern void send_message_to_frontend(ErrorData* edata); extern int SocketBackend(StringInfo inBuf); extern DestReceiver* printtup_create_DR(CommandDest dest); +extern void InitDolpinProtoIfNeeded(); ProtocolExtensionConfig* ListenConfig[MAXLISTEN]; @@ -2662,6 +2663,8 @@ int PostmasterMain(int argc, char* argv[]) ereport(FATAL, (errmsg("no socket created for listening"))); } + InitDolpinProtoIfNeeded(); + /* * Set up an on_proc_exit function that's charged with closing the sockets * again at postmaster shutdown. You might think we should have done this diff --git a/src/gausskernel/process/tcop/postgres.cpp b/src/gausskernel/process/tcop/postgres.cpp index 7ca0b86e6..38c81ee9c 100755 --- a/src/gausskernel/process/tcop/postgres.cpp +++ b/src/gausskernel/process/tcop/postgres.cpp @@ -991,6 +991,19 @@ void LoadDolphinIfNeeded() ExecuteFunctionIfExisted(DOLPHIN, LOAD_DOLPHIN); } } + +#define INIT_DOLPHIN_PROTO "init_dolphin_proto" +typedef void (*init_dolphin_proto_fn)(char *database_name); + +void InitDolpinProtoIfNeeded() +{ + if (IsFileExisted(DOLPHIN) && g_instance.attr.attr_network.enable_dolphin_proto) { + CFunInfo func = load_external_function(DOLPHIN, INIT_DOLPHIN_PROTO, false, false); + if (func.user_fn != NULL) { + ((init_dolphin_proto_fn)func.user_fn)(""); + } + } +} #endif /* From 39917e3171db0caee55d25cbee75eafa71593025 Mon Sep 17 00:00:00 2001 From: luo_zihao5524 Date: Wed, 26 Jul 2023 15:25:06 +0800 Subject: [PATCH 062/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dto=5Fchar=E5=87=BD?= =?UTF-8?q?=E6=95=B0=E5=AD=98=E5=9C=A8=E7=9A=84=E5=86=85=E5=AD=98=E8=B6=8A?= =?UTF-8?q?=E7=95=8C=E9=A3=8E=E9=99=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/utils/adt/formatting.cpp | 58 +++++++++++++++++---- 1 file changed, 47 insertions(+), 11 deletions(-) diff --git a/src/common/backend/utils/adt/formatting.cpp b/src/common/backend/utils/adt/formatting.cpp index b59afda08..4b9ea9451 100644 --- a/src/common/backend/utils/adt/formatting.cpp +++ b/src/common/backend/utils/adt/formatting.cpp @@ -3698,19 +3698,55 @@ static void DCH_to_char(FormatNode* node, bool is_interval, TmToChar* in, char* s += strlen(s); break; case DCH_RM: - if (!tm->tm_mon) - break; - rc = sprintf_s(s, len, "%*s", S_FM(n->suffix) ? 0 : -4, rm_months_upper[MONTHS_PER_YEAR - tm->tm_mon]); - securec_check_ss(rc, "\0", "\0"); - s += strlen(s); - break; case DCH_rm: - if (!tm->tm_mon) + /* + * For intervals, values like '12 month' will be reduced to 0 + * month and some years. These should be processed. + */ + if (!tm->tm_mon && !tm->tm_year) { break; - rc = sprintf_s(s, len, "%*s", S_FM(n->suffix) ? 0 : -4, rm_months_lower[MONTHS_PER_YEAR - tm->tm_mon]); - securec_check_ss(rc, "\0", "\0"); - s += strlen(s); - break; + } else { + int mon = 0; + char** months = NULL; + + if (n->key->id == DCH_RM) { + months = rm_months_upper; + } else { + months = rm_months_lower; + } + + /* + * Compute the position in the roman-numeral array. Note + * that the contents of the array are reversed, December + * being first and January last. + */ + if (tm->tm_mon == 0) { + /* + * This case is special, and tracks the case of full + * interval years. + */ + mon = tm->tm_year >= 0 ? 0 : MONTHS_PER_YEAR - 1; + } else if (tm->tm_mon < 0) { + /* + * Negative case. In this case, the calculation is + * reversed, where -1 means December, -2 November, + * etc. + */ + mon = -1 * (tm->tm_mon + 1); + } else { + /* + * Common case, with a strictly positive value. The + * position in the array matches with the value of + * tm_mon. + */ + mon = MONTHS_PER_YEAR - tm->tm_mon; + } + + rc = sprintf_s(s, len, "%*s", S_FM(n->suffix) ? 0 : -4, months[mon]); + securec_check_ss(rc, "\0", "\0"); + s += strlen(s); + break; + } case DCH_W: rc = sprintf_s(s, len, "%d", (tm->tm_mday - 1) / 7 + 1); securec_check_ss(rc, "\0", "\0"); From 4d33027523c5d25efdfe9acb1ef1feef780ba7a9 Mon Sep 17 00:00:00 2001 From: duzhuolin Date: Wed, 26 Jul 2023 19:26:27 +0800 Subject: [PATCH 063/304] =?UTF-8?q?pbe=E5=86=85=E5=AD=98=E7=BB=93=E7=82=B9?= =?UTF-8?q?=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/parser/analyze.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/backend/parser/analyze.cpp b/src/common/backend/parser/analyze.cpp index 176e8d370..12c7763ec 100644 --- a/src/common/backend/parser/analyze.cpp +++ b/src/common/backend/parser/analyze.cpp @@ -2630,7 +2630,7 @@ static bool shouldTransformStartWithStmt(ParseState* pstate, SelectStmt* stmt, Q * Back up the current select statement to be restored after query re-writing * for cases of START WITH CONNNECT BY under CREATE TABLE AS. */ - selectQuery->sql_statement = fetchSelectStmtFromCTAS((char*)pstate->p_sourcetext); + selectQuery->sql_statement = fetchSelectStmtFromCTAS((char*)pstrdup(pstate->p_sourcetext)); return true; } From 45073b3ecb7ddd1dbac8f98c2acd736429cf2373 Mon Sep 17 00:00:00 2001 From: duzhuolin Date: Wed, 26 Jul 2023 19:42:52 +0800 Subject: [PATCH 064/304] =?UTF-8?q?=E5=88=A0=E9=99=A4=E9=80=82=E5=90=88?= =?UTF-8?q?=E9=98=B2=E5=85=A5=E6=A0=A1=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/pgxc_clean/pgxc_clean.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/bin/pgxc_clean/pgxc_clean.cpp b/src/bin/pgxc_clean/pgxc_clean.cpp index 71c0688eb..2ea8c9d3f 100644 --- a/src/bin/pgxc_clean/pgxc_clean.cpp +++ b/src/bin/pgxc_clean/pgxc_clean.cpp @@ -2821,7 +2821,6 @@ static void parse_pgxc_clean_options(int argc, char* argv[]) case 'W': try_password_opt = TRI_YES; if (optarg != NULL) { - check_env_name_c(optarg); password = pg_strdup(optarg); rc = memset_s(optarg, strlen(optarg), 0, strlen(optarg)); securec_check_c(rc, "\0", "\0"); From 14723302392ef7e8422958aef465ce52d099ef6d Mon Sep 17 00:00:00 2001 From: gbzhangkai Date: Thu, 27 Jul 2023 09:28:59 +0800 Subject: [PATCH 065/304] overview fix --- src/bin/gs_loader/gs_loader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bin/gs_loader/gs_loader.cpp b/src/bin/gs_loader/gs_loader.cpp index 2a8398652..fd3e8b07c 100644 --- a/src/bin/gs_loader/gs_loader.cpp +++ b/src/bin/gs_loader/gs_loader.cpp @@ -127,7 +127,7 @@ int main(int argc, char **argv) // get the exe path for get shell script char abs_path[PATH_SIZE] = {'\0'}; int cnt = readlink("/proc/self/exe", abs_path, PATH_SIZE); - if (cnt <= -1 || cnt >= PATH_SIZE) { + if (cnt == -1 || cnt >= PATH_SIZE) { std::cout << "ERROR: can not find gs_loader path" << std::endl; return 0; } From d633e5c2a7f855586ebdedda4234eb5d207d36a6 Mon Sep 17 00:00:00 2001 From: lilong Date: Thu, 27 Jul 2023 10:38:22 +0800 Subject: [PATCH 066/304] =?UTF-8?q?=E5=87=BD=E6=95=B0CreateWeakPasswordDic?= =?UTF-8?q?tionary=E5=86=85=E9=83=A8=E5=88=A4=E6=96=ADrel=E4=BB=A3?= =?UTF-8?q?=E7=A0=81=E6=9C=89=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/optimizer/commands/tablecmds.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/gausskernel/optimizer/commands/tablecmds.cpp b/src/gausskernel/optimizer/commands/tablecmds.cpp index c747cba5b..337c62dd6 100755 --- a/src/gausskernel/optimizer/commands/tablecmds.cpp +++ b/src/gausskernel/optimizer/commands/tablecmds.cpp @@ -31270,12 +31270,6 @@ void CreateWeakPasswordDictionary(CreateWeakPasswordDictionaryStmt* stmt) } rel = heap_open(GsGlobalConfigRelationId, RowExclusiveLock); - if (!OidIsValid(rel)) { - ereport(ERROR, - (errcode(ERRCODE_SYSTEM_ERROR), - errmsg("could not open gs_global_config"))); - return; - } foreach (pwd_obj, stmt->weak_password_string_list) { Datum values[Natts_gs_global_config] = {0}; From 83aa08a9a3f5a600ae3df5d9b26ac2f32d0744fe Mon Sep 17 00:00:00 2001 From: duzhuolin Date: Thu, 27 Jul 2023 11:32:16 +0800 Subject: [PATCH 067/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=86=85=E5=AD=98?= =?UTF-8?q?=E8=B6=8A=E7=95=8C=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/include/postmaster/rbcleaner.h | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/include/postmaster/rbcleaner.h b/src/include/postmaster/rbcleaner.h index 604de6bd8..63bdceebe 100644 --- a/src/include/postmaster/rbcleaner.h +++ b/src/include/postmaster/rbcleaner.h @@ -233,7 +233,7 @@ static inline RbWorkerInfo *RbInitWorkerInfo(uint64 id, char *dbName) workerInfo->id = id; rc = strcpy_s(NameStr(workerInfo->dbName), NAMEDATALEN, dbName); - securec_check_c(rc, "\0", "\0"); + securec_check(rc, "\0", "\0"); InitLatch(&workerInfo->latch); @@ -274,7 +274,7 @@ static inline void RbMsgGetRes(uint64 id, PurgeMsgRes *localRes, bool reset = fa } else { errno_t rc = 0; rc = memcpy_s(localRes, sizeof(PurgeMsgRes), &rbMsg->res, sizeof(PurgeMsgRes)); - securec_check_c(rc, "\0", "\0"); + securec_check(rc, "\0", "\0"); } if (reset && RbMsgIsDone(rbMsg->res.status)) { rbMsg->id = RB_INVALID_MSGID; @@ -315,7 +315,7 @@ static inline void RbMsgSetStatistics(uint64 id, PurgeMsgRes *localRes) rbMsg->res.undefinedNum += localRes->undefinedNum; if (localRes->errMsg[0] != '\0') { rc = strncpy_s(rbMsg->res.errMsg, RB_MAX_ERRMSG_SIZE, localRes->errMsg, RB_MAX_ERRMSG_SIZE - 1); - securec_check_c(rc, "\0", "\0"); + securec_check(rc, "\0", "\0"); } SpinLockRelease(&rbMsg->mutex); } @@ -329,8 +329,9 @@ static inline void RbMsgSetStatusErr(uint32 id, PurgeMsgStatus status, SpinLockAcquire(&rbMsg->mutex); rbMsg->res.status = status; rbMsg->res.errCode = errCode; - rc = strcpy_s(rbMsg->res.errMsg, RB_MAX_ERRMSG_SIZE, errMsg); - securec_check_c(rc, "\0", "\0"); + rc = strncpy_s(rbMsg->res.errMsg, RB_MAX_ERRMSG_SIZE, errMsg, RB_MAX_ERRMSG_SIZE - 1); + securec_check(rc, "\0", "\0"); + if (setLatch) { SetLatch(&rbMsg->latch); } @@ -344,8 +345,9 @@ static inline void RbMsgSetStatusErrLocal(PurgeMsgRes *localRes, PurgeMsgStatus localRes->status = status; localRes->errCode = errCode; - rc = strcpy_s(localRes->errMsg, RB_MAX_ERRMSG_SIZE, errMsg); - securec_check_c(rc, "\0", "\0"); + rc = strncpy_s(localRes->errMsg, RB_MAX_ERRMSG_SIZE, errMsg, RB_MAX_ERRMSG_SIZE - 1); + securec_check(rc, "\0", "\0"); + } extern void RbCltPurgeSchema(Oid nspId); From 00cb06c98229999ebf5796b5cb1e34ed1957a940 Mon Sep 17 00:00:00 2001 From: duzhuolin Date: Thu, 27 Jul 2023 14:23:57 +0800 Subject: [PATCH 068/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dmask=E5=B8=A6?= =?UTF-8?q?=E8=BD=AC=E4=B9=89=E5=AD=97=E7=AC=A6=E5=AF=86=E7=A0=81=E6=97=B6?= =?UTF-8?q?=EF=BC=8C=E6=95=B0=E7=BB=84=E8=B6=8A=E7=95=8C=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/utils/error/elog.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/backend/utils/error/elog.cpp b/src/common/backend/utils/error/elog.cpp index 697fa28e0..d2dc5fbfc 100644 --- a/src/common/backend/utils/error/elog.cpp +++ b/src/common/backend/utils/error/elog.cpp @@ -5036,7 +5036,7 @@ static char* mask_Password_internal(const char* query_string) int lengthOfQuote = 0; int yyvalLen = (yylval.str != NULL) ? (int)strlen(yylval.str) : 0; for (int len = 0; len < length[i]; len++) { - if (len < yyvalLen && (yylval.str[len] == '\'')) { + if ((yylval.str != NULL) && len < (int)strlen(yylval.str) && (yylval.str[len] == '\'')) { lengthOfQuote++; } } From 296918b6aff488b3596020f31aa4a2068abbe249 Mon Sep 17 00:00:00 2001 From: duzhuolin Date: Thu, 27 Jul 2023 18:01:45 +0800 Subject: [PATCH 069/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8DGSQL=E5=86=85?= =?UTF-8?q?=E5=AD=98=E6=B3=84=E9=9C=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/psql/copy.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/bin/psql/copy.cpp b/src/bin/psql/copy.cpp index 0f7bb8b29..401616cc9 100644 --- a/src/bin/psql/copy.cpp +++ b/src/bin/psql/copy.cpp @@ -768,6 +768,8 @@ bool handleCopyIn(PGconn* conn, FILE* copystream, bool isbinary) if (strncmp((const char*)pset.decryptInfo.decryptBuff, "\\.\n", strlen("\\.\n")) == 0 || strncmp((const char*)pset.decryptInfo.decryptBuff, "\\.\r\n", strlen("\\.\n")) == 0) { copydone = true; + free(pset.decryptInfo.decryptBuff); + pset.decryptInfo.decryptBuff = NULL; break; } @@ -777,6 +779,8 @@ bool handleCopyIn(PGconn* conn, FILE* copystream, bool isbinary) if (PQputCopyData(conn, (const char*)pset.decryptInfo.decryptBuff, linelen) <= 0) { OK = false; copydone = true; + free(pset.decryptInfo.decryptBuff); + pset.decryptInfo.decryptBuff = NULL; break; } From 1bb8db709f1336bdd207ed64e5b8e72e8f7301fd Mon Sep 17 00:00:00 2001 From: Julong-Li <584147810@qq.com> Date: Tue, 25 Jul 2023 10:36:43 +0800 Subject: [PATCH 070/304] =?UTF-8?q?union=20all=20=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E5=AD=90=E8=AF=AD=E5=8F=A5=E6=8E=92=E5=BA=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/gs_guc/cluster_guc.conf | 1 + src/common/backend/utils/misc/guc/guc_sql.cpp | 11 +++++ .../optimizer/prep/prepnonjointree.cpp | 18 +++++--- .../knl/knl_guc/knl_session_attr_sql.h | 1 + .../expected/test_union_all_orderby.out | 43 +++++++++++++++++++ .../regress/output/recovery_2pc_tools.source | 1 + src/test/regress/parallel_schedule0 | 3 +- src/test/regress/parallel_schedule0C | 4 +- .../regress/sql/test_union_all_orderby.sql | 15 +++++++ 9 files changed, 89 insertions(+), 8 deletions(-) create mode 100644 src/test/regress/expected/test_union_all_orderby.out create mode 100644 src/test/regress/sql/test_union_all_orderby.sql diff --git a/src/bin/gs_guc/cluster_guc.conf b/src/bin/gs_guc/cluster_guc.conf index 986e134d3..c59516547 100755 --- a/src/bin/gs_guc/cluster_guc.conf +++ b/src/bin/gs_guc/cluster_guc.conf @@ -115,6 +115,7 @@ commit_siblings|int|0,1000|NULL|NULL| config_file|string|0,0|NULL|NULL| connection_alarm_rate|real|0,1|NULL|NULL| constraint_exclusion|enum|partition,on,off,true,false,yes,no,1,0|NULL|NULL| +enable_union_all_subquery_orderby|bool|0,0|NULL|NULL| convert_string_to_digit|bool|0,0|NULL|Please don't modify this parameter which will change the type conversion rule and may lead to unpredictable behavior!| cost_param|int|0,2147483647|NULL|NULL| cpu_collect_timer|int|1,2147483647|NULL|NULL| diff --git a/src/common/backend/utils/misc/guc/guc_sql.cpp b/src/common/backend/utils/misc/guc/guc_sql.cpp index eea75db5a..a4d460d8d 100755 --- a/src/common/backend/utils/misc/guc/guc_sql.cpp +++ b/src/common/backend/utils/misc/guc/guc_sql.cpp @@ -473,6 +473,17 @@ static void InitSqlConfigureNamesBool() NULL, NULL, NULL}, + {{"enable_union_all_subquery_orderby", + PGC_USERSET, + NODE_SINGLENODE, + QUERY_TUNING_METHOD, + gettext_noop("Enable union all to order subquery."), + NULL}, + &u_sess->attr.attr_sql.enable_union_all_subquery_orderby, + false, + NULL, + NULL, + NULL}, {{"enable_global_stats", PGC_SUSET, NODE_ALL, diff --git a/src/gausskernel/optimizer/prep/prepnonjointree.cpp b/src/gausskernel/optimizer/prep/prepnonjointree.cpp index 505a4e7ba..328cdfe51 100755 --- a/src/gausskernel/optimizer/prep/prepnonjointree.cpp +++ b/src/gausskernel/optimizer/prep/prepnonjointree.cpp @@ -2178,9 +2178,12 @@ static void reduce_orderby_recurse(Query* query, Node* jtnode, bool reduce) } } else if (IsA(jtnode, SetOperationStmt)) { SetOperationStmt* op = (SetOperationStmt*)jtnode; - - reduce_orderby_recurse(query, op->larg, true); - reduce_orderby_recurse(query, op->rarg, true); + bool need_reduce = true; + if (op->op == SETOP_UNION && op->all && u_sess->attr.attr_sql.enable_union_all_subquery_orderby) { + need_reduce = reduce; + } + reduce_orderby_recurse(query, op->larg, need_reduce); + reduce_orderby_recurse(query, op->rarg, need_reduce); } return; @@ -2243,8 +2246,13 @@ void reduce_orderby(Query* query, bool reduce) /* If there has setop, it should optimize orderby clause. */ if (query->setOperations) { - reduce_orderby_recurse(query, ((SetOperationStmt*)query->setOperations)->larg, true); - reduce_orderby_recurse(query, ((SetOperationStmt*)query->setOperations)->rarg, true); + bool need_reduce = true; + SetOperationStmt *setop_stmt = (SetOperationStmt *)query->setOperations; + if (setop_stmt->op == SETOP_UNION && setop_stmt->all && u_sess->attr.attr_sql.enable_union_all_subquery_orderby) { + need_reduce = reduce; + } + reduce_orderby_recurse(query, ((SetOperationStmt*)query->setOperations)->larg, need_reduce); + reduce_orderby_recurse(query, ((SetOperationStmt*)query->setOperations)->rarg, need_reduce); } } diff --git a/src/include/knl/knl_guc/knl_session_attr_sql.h b/src/include/knl/knl_guc/knl_session_attr_sql.h index c5cf08952..74e69d666 100644 --- a/src/include/knl/knl_guc/knl_session_attr_sql.h +++ b/src/include/knl/knl_guc/knl_session_attr_sql.h @@ -63,6 +63,7 @@ typedef struct knl_session_attr_sql { bool enable_indexonlyscan; bool enable_bitmapscan; bool force_bitmapand; + bool enable_union_all_subquery_orderby; bool enable_parallel_ddl; bool enable_tidscan; bool enable_sort; diff --git a/src/test/regress/expected/test_union_all_orderby.out b/src/test/regress/expected/test_union_all_orderby.out new file mode 100644 index 000000000..a0da822e9 --- /dev/null +++ b/src/test/regress/expected/test_union_all_orderby.out @@ -0,0 +1,43 @@ +set enable_union_all_subquery_orderby=on; +create table tb1(a int); +insert into tb1 values(1); +insert into tb1 values(3); +insert into tb1 values(2); +create table tb2(a int); +insert into tb2 values(5); +insert into tb2 values(4); +create table tb3(a int); +insert into tb3 values(7); +insert into tb3 values(6); +(select * from tb1 order by a) union all (select * from tb2 order by a); + a +--- + 1 + 2 + 3 + 4 + 5 +(5 rows) + +(select * from tb1 order by a) union all (select * from tb2 order by a desc); + a +--- + 1 + 2 + 3 + 5 + 4 +(5 rows) + +(select * from tb1 order by a) union all (select * from tb2 order by a) union all (select * from tb3 order by a); + a +--- + 1 + 2 + 3 + 4 + 5 + 6 + 7 +(7 rows) + diff --git a/src/test/regress/output/recovery_2pc_tools.source b/src/test/regress/output/recovery_2pc_tools.source index c491ca40a..63bf34624 100644 --- a/src/test/regress/output/recovery_2pc_tools.source +++ b/src/test/regress/output/recovery_2pc_tools.source @@ -331,6 +331,7 @@ select name,vartype,unit,min_val,max_val from pg_settings where name <> 'qunit_c enable_tde | bool | | | enable_thread_pool | bool | | | enable_tidscan | bool | | | + enable_union_all_subquery_orderby | bool | | | enable_upgrade_merge_lock_mode | bool | | | enable_user_metric_persistent | bool | | | enable_ustore | bool | | | diff --git a/src/test/regress/parallel_schedule0 b/src/test/regress/parallel_schedule0 index f7f9846b4..885c81222 100644 --- a/src/test/regress/parallel_schedule0 +++ b/src/test/regress/parallel_schedule0 @@ -905,7 +905,7 @@ test: vec_sonic_hashjoin_number_nospill test: timeout test: dml test: hashfilter hashfilter_1 -test: reduce_orderby +test: reduce_orderby test_union_all_orderby #test: backtrace_log #test: bulkload_start test: bulkload_parallel_test_2 bulkload_parallel_test_3 @@ -1098,3 +1098,4 @@ test: user_host_test test: enable_expr_fusion_flatten # test for on update timestamp and generated column test: on_update_session1 on_update_session2 + diff --git a/src/test/regress/parallel_schedule0C b/src/test/regress/parallel_schedule0C index 3bcbb5177..a7ef5ba08 100644 --- a/src/test/regress/parallel_schedule0C +++ b/src/test/regress/parallel_schedule0C @@ -130,7 +130,7 @@ test: mysql_function_clearup cost_model test: base_update test: seqscan_fusion -test: union_null_01 fulljoin_rewrite +test: union_null_01 fulljoin_rewrite test_union_all_orderby # var selectivity test: var_eq_const_selectivity @@ -183,4 +183,4 @@ test: slow_sql test: user_host_test # test for new_expr_by_flatten -test: enable_expr_fusion_flatten \ No newline at end of file +test: enable_expr_fusion_flatten diff --git a/src/test/regress/sql/test_union_all_orderby.sql b/src/test/regress/sql/test_union_all_orderby.sql new file mode 100644 index 000000000..d239a68d2 --- /dev/null +++ b/src/test/regress/sql/test_union_all_orderby.sql @@ -0,0 +1,15 @@ +set enable_union_all_subquery_orderby=on; +create table tb1(a int); +insert into tb1 values(1); +insert into tb1 values(3); +insert into tb1 values(2); +create table tb2(a int); +insert into tb2 values(5); +insert into tb2 values(4); +create table tb3(a int); +insert into tb3 values(7); +insert into tb3 values(6); + +(select * from tb1 order by a) union all (select * from tb2 order by a); +(select * from tb1 order by a) union all (select * from tb2 order by a desc); +(select * from tb1 order by a) union all (select * from tb2 order by a) union all (select * from tb3 order by a); \ No newline at end of file From b9f94537dc9f00e02336e57f7956790e0f848b51 Mon Sep 17 00:00:00 2001 From: xuxinxin Date: Thu, 27 Jul 2023 19:29:32 +0800 Subject: [PATCH 071/304] =?UTF-8?q?=E6=8E=A8=E8=BF=9Bdsscommit=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/ddes/ddes_commit_id | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gausskernel/ddes/ddes_commit_id b/src/gausskernel/ddes/ddes_commit_id index afde56012..e1fdc9cb9 100644 --- a/src/gausskernel/ddes/ddes_commit_id +++ b/src/gausskernel/ddes/ddes_commit_id @@ -1,2 +1,2 @@ dms_commit_id=a9d399af041ae5491149fdb7c58e5115da3065b2 -dss_commit_id=7d0dcfc24d42edcf62b91ef6334fca103d21643a +dss_commit_id=8a4a7a5fb2bbe63b7596709e474459ff24359c08 From 67a89d6788e0c1bcf96477f543ec3f856a644260 Mon Sep 17 00:00:00 2001 From: zhang_xubo <2578876417@qq.com> Date: Thu, 27 Jul 2023 19:41:45 +0800 Subject: [PATCH 072/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8Ddocker=E6=9E=84?= =?UTF-8?q?=E5=BB=BA=E7=89=88=E6=9C=AC=E6=A0=A1=E9=AA=8C=E8=A7=84=E5=88=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker/dockerfiles/buildDockerImage.sh | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/docker/dockerfiles/buildDockerImage.sh b/docker/dockerfiles/buildDockerImage.sh index 9c8b84cb8..3cdcc067f 100644 --- a/docker/dockerfiles/buildDockerImage.sh +++ b/docker/dockerfiles/buildDockerImage.sh @@ -67,16 +67,16 @@ check_docker_version() { echo "Checking Docker version." DOCKER_VERSION=$(docker version --format '{{.Server.Version | printf "%.5s" }}'|| exit 0) # Remove dot in Docker version - DOCKER_VERSION=${DOCKER_VERSION//./} + docker_version_major=$(echo $DOCKER_VERSION | awk -F . '{print $1}') if [ -z "$DOCKER_VERSION" ]; then # docker could be aliased to podman and errored out (https://github.com/containers/libpod/pull/4608) - checkPodmanVersion - elif [ "$DOCKER_VERSION" -lt "${MIN_DOCKER_VERSION//./}" ]; then - echo "Docker version is below the minimum required version $MIN_DOCKER_VERSION" + echo "Please check if docker is installed." && exit 1 + elif [ "$docker_version_major" -lt "${MIN_DOCKER_VERSION_MAJOR}" ]; then + echo "Docker version is below the minimum required version $MIN_DOCKER_VERSION_MAJOR.$MIN_DOCKER_VERSION_MINOR" echo "Please upgrade your Docker installation to proceed." exit 1; - fi; + fi } ############## @@ -87,7 +87,8 @@ check_docker_version() { VERSION="5.0.0" SKIPCHECKSUM=0 DOCKEROPS="" -MIN_DOCKER_VERSION="17.09" +MIN_DOCKER_VERSION_MAJOR="17" +MIN_DOCKER_VERSION_MINOR="09" arch=$(case $(uname -m) in i386) echo "386" ;; i686) echo "386" ;; x86_64) echo "amd64";; aarch64)echo "arm64";; esac) if [ "${arch}" = "amd64" ]; then DOCKERFILE="dockerfile_amd" @@ -129,12 +130,6 @@ done check_docker_version - -# Which Dockerfile should be used? -if [ "$VERSION" == "12.1.0.2" ] || [ "$VERSION" == "11.2.0.2" ] || [ "$VERSION" == "18.4.0" ]; then - DOCKERFILE="$DOCKERFILE" -fi; - # openGauss Database Image Name IMAGE_NAME="opengauss:$VERSION" From 5cc110ed9626af2ec3b13f4eb62e7ec967e9f174 Mon Sep 17 00:00:00 2001 From: duzhuolin Date: Thu, 27 Jul 2023 13:50:07 +0000 Subject: [PATCH 073/304] update src/common/backend/utils/error/elog.cpp. Signed-off-by: duzhuolin --- src/common/backend/utils/error/elog.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/backend/utils/error/elog.cpp b/src/common/backend/utils/error/elog.cpp index d2dc5fbfc..24fa62bf1 100644 --- a/src/common/backend/utils/error/elog.cpp +++ b/src/common/backend/utils/error/elog.cpp @@ -5036,7 +5036,7 @@ static char* mask_Password_internal(const char* query_string) int lengthOfQuote = 0; int yyvalLen = (yylval.str != NULL) ? (int)strlen(yylval.str) : 0; for (int len = 0; len < length[i]; len++) { - if ((yylval.str != NULL) && len < (int)strlen(yylval.str) && (yylval.str[len] == '\'')) { + if ((yylval.str != NULL) && len < yyvalLen && (yylval.str[len] == '\'')) { lengthOfQuote++; } } From 1ce8473ffcc70ba346af69f99b5bf4a07682f35c Mon Sep 17 00:00:00 2001 From: duzhuolin Date: Thu, 27 Jul 2023 21:58:54 +0800 Subject: [PATCH 074/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E4=B8=80=E7=BA=A7?= =?UTF-8?q?=E5=88=86=E5=8C=BAustore=E8=A1=A8exchange=20partition=E5=90=8E?= =?UTF-8?q?=EF=BC=8Cupdate=E6=95=B0=E6=8D=AE=E5=A4=B1=E8=B4=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/catalog/index.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/common/backend/catalog/index.cpp b/src/common/backend/catalog/index.cpp index 7e99bbf96..415c9a0ee 100644 --- a/src/common/backend/catalog/index.cpp +++ b/src/common/backend/catalog/index.cpp @@ -6590,10 +6590,8 @@ void ScanPartitionInsertIndex(Relation partTableRel, Relation partRel, const Lis ListCell* cell = NULL; ListCell* cell1 = NULL; EState* estate = NULL; - TupleDesc tupleDesc = NULL; TupleTableSlot* slot = NULL; - tupleDesc = partRel->rd_att; if (PointerIsValid(indexRelList)) { estate = CreateExecutorState(); @@ -6614,8 +6612,9 @@ void ScanPartitionInsertIndex(Relation partTableRel, Relation partRel, const Lis Relation indexRel = (Relation)lfirst(cell); IndexInfo* indexInfo = static_cast(lfirst(cell1)); - Datum values[tupleDesc->natts]; - bool isNull[tupleDesc->natts]; + Datum values[INDEX_MAX_KEYS]; + bool isNull[INDEX_MAX_KEYS]; + bool estateIsNotNull = false; ItemPointer t_ctid = tableam_tops_get_t_self(partTableRel, tuple); From 09654bc74dfddf603ad4900b31e5a12bef1b8e81 Mon Sep 17 00:00:00 2001 From: duzhuolin Date: Thu, 27 Jul 2023 22:17:53 +0800 Subject: [PATCH 075/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E8=8A=82=E7=82=B9?= =?UTF-8?q?=E6=9C=AA=E9=87=8A=E6=94=BE=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/pg_dump/pg_dump.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/bin/pg_dump/pg_dump.cpp b/src/bin/pg_dump/pg_dump.cpp index 87616b0af..f7827fccd 100644 --- a/src/bin/pg_dump/pg_dump.cpp +++ b/src/bin/pg_dump/pg_dump.cpp @@ -3758,6 +3758,7 @@ static void dumpClientGlobalKeys(Archive *fout, const Oid nspoid, const char *ns rc = memcpy_s(algorithm, MAX_CLIENT_KEYS_PARAM_SIZE, unescaped_data, unescapedDataSize); securec_check_c(rc, "", ""); } + PQfreemem(unescaped_data); } appendPQExpBuffer(createClientGlobalKeysQry, "CREATE CLIENT MASTER KEY %s.%s ", nspname, fmtId(global_key_name)); @@ -3897,6 +3898,7 @@ static void dumpColumnEncryptionKeys(Archive *fout, const Oid nspoid, const char rc = memcpy_s(key_algorithm_str, MAX_CLIENT_KEYS_PARAM_SIZE, unescaped_data, unescapedDataSize); } securec_check_c(rc, "", ""); + PQfreemem(unescaped_data); } unsigned char *deprocessed_cek = NULL; @@ -3939,6 +3941,7 @@ static void dumpColumnEncryptionKeys(Archive *fout, const Oid nspoid, const char rc = memcpy_s(algorithm, MAX_CLIENT_KEYS_PARAM_SIZE, unescaped_data, unescapedDataSize); securec_check_c(rc, "", ""); } + PQfreemem(unescaped_data); } appendPQExpBuffer(createColumnEncryptionKeysQry, "CREATE COLUMN ENCRYPTION KEY %s.%s ", nspname, column_key_name); From e83e4397e60cb81d2865bf77153aa23011208ec4 Mon Sep 17 00:00:00 2001 From: duzhuolin Date: Thu, 27 Jul 2023 22:35:43 +0800 Subject: [PATCH 076/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=86=85=E5=AD=98?= =?UTF-8?q?=E6=B3=84=E6=BC=8F=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/pg_rewind/fetch.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/bin/pg_rewind/fetch.cpp b/src/bin/pg_rewind/fetch.cpp index 03ad5b58f..19d818eba 100755 --- a/src/bin/pg_rewind/fetch.cpp +++ b/src/bin/pg_rewind/fetch.cpp @@ -123,6 +123,8 @@ BuildErrorCode libpqGetParameters(void) } str2 = run_simple_query("SHOW enable_incremental_checkpoint"); if (str2 == NULL) { + pg_free(str); + str = NULL; return BUILD_FATAL; } if (strcmp(str, "on") != 0 && strcmp(str2, "on") != 0) { From 58b8cf417822d71b79bb6a2abb71c6cac0212fc0 Mon Sep 17 00:00:00 2001 From: duzhuolin Date: Thu, 27 Jul 2023 22:47:29 +0800 Subject: [PATCH 077/304] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E5=86=85=E5=AD=98?= =?UTF-8?q?=E6=B3=84=E6=BC=8F=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/cbb/workload/statctl.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/gausskernel/cbb/workload/statctl.cpp b/src/gausskernel/cbb/workload/statctl.cpp index 798cbe9e8..a62d0803c 100644 --- a/src/gausskernel/cbb/workload/statctl.cpp +++ b/src/gausskernel/cbb/workload/statctl.cpp @@ -1182,6 +1182,8 @@ void WLMCleanUpNodeInternal(const Qid* qid) /* No threads already, remove the info from the hash table. */ if (info->threadCount <= 0) { + pfree_ext(info->qband); + pfree_ext(info->statement); hash_search(g_instance.wlm_cxt->stat_manager.collect_info_hashtbl, qid, HASH_REMOVE, NULL); } } @@ -4190,8 +4192,10 @@ bool WLMInsertCollectInfoIntoHashTable(void) USE_MEMORY_CONTEXT(g_instance.wlm_cxt->query_resource_track_mcxt); if (u_sess->attr.attr_resource.query_band) { + pfree_ext(pDNodeInfo->qband); pDNodeInfo->qband = pstrdup(u_sess->attr.attr_resource.query_band); } + pfree_ext(pDNodeInfo->statement); pDNodeInfo->statement = pstrdup(t_thrd.wlm_cxt.collect_info->sdetail.statement); From 0c32ca78ef5053bd09352b6042eaa8084133ec11 Mon Sep 17 00:00:00 2001 From: duzhuolin Date: Thu, 27 Jul 2023 23:07:27 +0800 Subject: [PATCH 078/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E4=BD=BF=E7=94=A8?= =?UTF-8?q?=E8=8A=82=E7=82=B9=E6=9C=AA=E9=87=8A=E6=94=BE=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/optimizer/commands/prepare.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/gausskernel/optimizer/commands/prepare.cpp b/src/gausskernel/optimizer/commands/prepare.cpp index d2fdeaa37..967119f3c 100755 --- a/src/gausskernel/optimizer/commands/prepare.cpp +++ b/src/gausskernel/optimizer/commands/prepare.cpp @@ -1556,6 +1556,7 @@ void DropDatanodeStatement(const char* stmt_name) (void*)hash_search(u_sess->pcache_cxt.datanode_queries, entry->stmt_name, HASH_REMOVE, NULL); if (!ENABLE_CN_GPC) ExecCloseRemoteStatement(stmt_name, nodelist); + list_free_ext(nodelist); } } From fc87e3ed6b159a823b27c4564a8d6709f373faf3 Mon Sep 17 00:00:00 2001 From: zhangchao Date: Fri, 28 Jul 2023 09:37:58 +0800 Subject: [PATCH 079/304] =?UTF-8?q?=E6=94=AF=E6=8C=81ldap=E8=AE=A4?= =?UTF-8?q?=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/interfaces/libpq/fe-auth.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/common/interfaces/libpq/fe-auth.cpp b/src/common/interfaces/libpq/fe-auth.cpp index c28bca692..fc56b5099 100644 --- a/src/common/interfaces/libpq/fe-auth.cpp +++ b/src/common/interfaces/libpq/fe-auth.cpp @@ -1040,6 +1040,9 @@ static int pg_password_sendauth(PGconn* conn, const char* password, AuthRequest break; } + case AUTH_REQ_PASSWORD: + pwd_to_send = password; + break; /* * Notice: Authentication of send password directly are not currently supported. * need to: We remove the branch of AUTH_REQ_PASSWORD here for both implication and @@ -1208,6 +1211,7 @@ int pg_fe_sendauth(AuthRequest areq, PGconn* conn) case AUTH_REQ_MD5: case AUTH_REQ_MD5_SHA256: + case AUTH_REQ_PASSWORD: case AUTH_REQ_SHA256: #ifdef ENABLE_LITE_MODE case AUTH_REQ_SHA256_RFC: From cf1887ede511a56167982cdb2d39fcb4749cee86 Mon Sep 17 00:00:00 2001 From: duzhuolin Date: Fri, 28 Jul 2023 09:39:50 +0800 Subject: [PATCH 080/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8DPushOverrideSearchPat?= =?UTF-8?q?h=E5=86=85=E5=AD=98=E6=B3=84=E9=9C=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/catalog/namespace.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/common/backend/catalog/namespace.cpp b/src/common/backend/catalog/namespace.cpp index 74e1c11d4..c03f43c11 100644 --- a/src/common/backend/catalog/namespace.cpp +++ b/src/common/backend/catalog/namespace.cpp @@ -3940,6 +3940,8 @@ void PushOverrideSearchPath(OverrideSearchPath* newpath, bool inProcedure) break; } } + pfree_ext(rawname); + list_free_ext(namelist); } /* From 87237a07f353012eb64cf3138d9f3cc60b5207eb Mon Sep 17 00:00:00 2001 From: duzhuolin Date: Fri, 28 Jul 2023 09:45:06 +0800 Subject: [PATCH 081/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8Djobschd=5Fsigusr2=5Fh?= =?UTF-8?q?andler=E4=BF=A1=E5=8F=B7=E5=A4=84=E7=90=86=E5=87=BD=E6=95=B0?= =?UTF-8?q?=E6=89=93=E5=8D=B0=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/process/job/job_scheduler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gausskernel/process/job/job_scheduler.cpp b/src/gausskernel/process/job/job_scheduler.cpp index e7e651038..3f6073001 100755 --- a/src/gausskernel/process/job/job_scheduler.cpp +++ b/src/gausskernel/process/job/job_scheduler.cpp @@ -520,7 +520,7 @@ static void jobschd_sighup_handler(SIGNAL_ARGS) static void jobschd_sigusr2_handler(SIGNAL_ARGS) { int save_errno = errno; - elog(LOG, "Job scheduler received sigusr2 when job worker startup failed."); + write_stderr("Job scheduler received sigusr2 when job worker startup failed."); t_thrd.job_cxt.got_SIGUSR2 = true; if (t_thrd.proc) { From fa1ea53cf17ba6624e1371e91be48bbfe6acc132 Mon Sep 17 00:00:00 2001 From: zhangchao Date: Fri, 28 Jul 2023 11:18:26 +0800 Subject: [PATCH 082/304] =?UTF-8?q?pg=5Fregress.cpp=E5=86=85=E5=AD=98?= =?UTF-8?q?=E6=B3=84=E9=9C=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/test/regress/pg_regress.cpp | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/src/test/regress/pg_regress.cpp b/src/test/regress/pg_regress.cpp index 0f523fafd..090b937ce 100644 --- a/src/test/regress/pg_regress.cpp +++ b/src/test/regress/pg_regress.cpp @@ -611,6 +611,7 @@ static int regrGetServPid(char* pcBuff, unsigned int uiBuffLen, #else if (!getcwd(cwd, MAXPGPATH)) { fprintf(stderr, _("\n Get current dir fail.\n")); + free((char*)data_folder); return -1; } if (strlen(cwd) + strlen("/tmp_check/") + strlen(data_folder) + strlen("/postmaster.pid") >= uiBuffLen) { @@ -620,6 +621,7 @@ static int regrGetServPid(char* pcBuff, unsigned int uiBuffLen, "length needed: %lu.\n"), uiBuffLen, (strlen(cwd) + strlen("/") + strlen("tmp_check/") + strlen(data_folder) + strlen("/postmaster.pid") + 1)); + free((char*)data_folder); return REGR_ERRCODE_BUFF_NOT_ENOUGH; } @@ -1333,6 +1335,7 @@ static void start_gtm(void) /* Save static PID number */ myinfo.gtm_pid = node_pid; + free((char*)data_folder); } /* Start single datanode for test */ @@ -1740,6 +1743,7 @@ static void* thread_initdb(void* arg) static void start_thread(thread_desc* thread, bool is_cn, bool is_standby, int i) { char* data_folder = get_node_info_name(i, is_cn ? COORD : DATANODE, false); + char* node_name = get_node_info_name(i, is_cn ? COORD : DATANODE, true); char** args = thread->args; int thi = thread->thi; pthread_t* thd = &(thread->thd[thi]); @@ -1750,7 +1754,7 @@ static void start_thread(thread_desc* thread, bool is_cn, bool is_standby, int i SYSTEMQUOTE "\"%s/gs_initdb\" --nodename %s %s -w \"gauss@123\" -D \"%s/%s%s\" -L \"%s\" --noclean%s%s > " "\"%s/log/initdb%d.log\" 2>&1" SYSTEMQUOTE, bindir, - (char*)get_node_info_name(i, is_cn ? COORD : DATANODE, true), + node_name, init_database ? "-U upcheck" : "", temp_install, data_folder, @@ -1773,6 +1777,7 @@ static void start_thread(thread_desc* thread, bool is_cn, bool is_standby, int i thread->thi++; free(data_folder); + free(node_name); } /* @@ -1986,12 +1991,13 @@ static void initdb_node_info(bool standby) char buf[MAXPGPATH * 4]; char* data_folder = get_node_info_name(i, DATANODE, false); + char* node_name = get_node_info_name(i, DATANODE, true; (void)snprintf(buf, sizeof(buf), SYSTEMQUOTE "\"%s/gs_initdb\" --nodename %s %s -w \"gauss@123\" -D \"%s/%s_standby\" -L \"%s\" " "--noclean%s%s > \"%s/log/initdb.log\" 2>&1" SYSTEMQUOTE, bindir, - (char*)get_node_info_name(i, DATANODE, true), + node_name, init_database ? "-U upcheck" : "", temp_install, data_folder, @@ -2008,6 +2014,7 @@ static void initdb_node_info(bool standby) exit_nicely(2); } free(data_folder); + free(node_name); } } @@ -3191,10 +3198,18 @@ static void convertSourcefilesIn(char* pcSourceSubdir, char* pcDestDir, char* pc replace_string(line, "@libdir@", dlpath); replace_string(line, "@DLSUFFIX@", DLSUFFIX); replace_string(line, "@gsqldir@", psqldir); - replace_string(line, "@datanode1@", get_node_info_name(0, DATANODE, true)); - replace_string(line, "@datanode2@", get_node_info_name(1, DATANODE, true)); - replace_string(line, "@coordinator1@", get_node_info_name(0, COORD, true)); - replace_string(line, "@coordinator2@", get_node_info_name(1, COORD, true)); + char* node_name = get_node_info_name(0, DATANODE, true); + replace_string(line, "@datanode1@", node_name); + free(node_name); + node_name = get_node_info_name(1, DATANODE, true); + replace_string(line, "@datanode2@", node_name); + free(node_name); + node_name=get_node_info_name(0, COORD, true); + replace_string(line, "@coordinator1@", node_name); + free(node_name); + node_name = get_node_info_name(1, COORD, true); + replace_string(line, "@coordinator2@", node_name); + free(node_name); replace_string(line, "@pgbench_dir@", pgbenchdir); replace_string(line, "@client_logic_hook@", client_logic_hook); char* ptr = GetStartNodeCmdString(0, DATANODE); From 78f4c059c8e4e0b17edc98026320492dfe75a4ca Mon Sep 17 00:00:00 2001 From: congzhou2603 Date: Fri, 28 Jul 2023 11:00:59 +0800 Subject: [PATCH 083/304] =?UTF-8?q?=E4=BF=AE=E6=94=B9compresstype=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E8=8C=83=E5=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/storage/access/common/reloptions.cpp | 2 +- .../expected/row_compression/alter_compress_params.out | 7 ++++--- .../expected/row_compression/unsupported_feature.out | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/gausskernel/storage/access/common/reloptions.cpp b/src/gausskernel/storage/access/common/reloptions.cpp index fa2d63c6f..28163d71e 100644 --- a/src/gausskernel/storage/access/common/reloptions.cpp +++ b/src/gausskernel/storage/access/common/reloptions.cpp @@ -238,7 +238,7 @@ static relopt_int intRelOpts[] = { 0, 1, 32 }, {{ "compress_level", "Level of page compression.", RELOPT_KIND_HEAP | RELOPT_KIND_BTREE}, 0, -31, 31}, - {{ "compresstype", "compress type (none, pglz or zstd or pgzstd).", RELOPT_KIND_HEAP | RELOPT_KIND_BTREE}, 0, 0, 3}, + {{ "compresstype", "compress type (none, pglz or zstd. pgzstd isn't available now).", RELOPT_KIND_HEAP | RELOPT_KIND_BTREE}, 0, 0, 2}, {{ "compress_chunk_size", "Size of chunk to store compressed page.", RELOPT_KIND_HEAP | RELOPT_KIND_BTREE}, BLCKSZ / 2, BLCKSZ / 16, diff --git a/src/test/regress/expected/row_compression/alter_compress_params.out b/src/test/regress/expected/row_compression/alter_compress_params.out index 260e565c1..84b71353f 100644 --- a/src/test/regress/expected/row_compression/alter_compress_params.out +++ b/src/test/regress/expected/row_compression/alter_compress_params.out @@ -31,13 +31,13 @@ CREATE TABLE alter_compress_params_schema.uncompress_astore_to_compresstype_5 (i INSERT INTO alter_compress_params_schema.uncompress_astore_to_compresstype_5 SELECT generate_series(1,5), 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'; ALTER TABLE alter_compress_params_schema.uncompress_astore_to_compresstype_5 SET (compresstype = 5); -- fail ERROR: value 5 out of bounds for option "compresstype" -DETAIL: Valid values are between "0" and "3". +DETAIL: Valid values are between "0" and "2". DROP TABLE alter_compress_params_schema.uncompress_astore_to_compresstype_5; CREATE TABLE alter_compress_params_schema.uncompress_astore_to_compresstype__1 (id int, value varchar); INSERT INTO alter_compress_params_schema.uncompress_astore_to_compresstype__1 SELECT generate_series(1,5), 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'; ALTER TABLE alter_compress_params_schema.uncompress_astore_to_compresstype__1 SET (compresstype = -1); -- fail ERROR: value -1 out of bounds for option "compresstype" -DETAIL: Valid values are between "0" and "3". +DETAIL: Valid values are between "0" and "2". DROP TABLE alter_compress_params_schema.uncompress_astore_to_compresstype__1; CREATE TABLE alter_compress_params_schema.uncompress_astore_to_compresstype_2_cl_32 (id int, value varchar); INSERT INTO alter_compress_params_schema.uncompress_astore_to_compresstype_2_cl_32 SELECT generate_series(1,5), 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'; @@ -98,7 +98,8 @@ DROP TABLE alter_compress_params_schema.uncompress_astore_to_compresstype_1_ccs_ CREATE TABLE alter_compress_params_schema.uncompress_to_compresstype_3 (id int, value varchar); INSERT INTO alter_compress_params_schema.uncompress_to_compresstype_3 SELECT generate_series(1,5), 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'; ALTER TABLE alter_compress_params_schema.uncompress_to_compresstype_3 SET (compresstype = 3); -- fail -ERROR: row-compression feature current not support algorithm is PGZSTD. +ERROR: value 3 out of bounds for option "compresstype" +DETAIL: Valid values are between "0" and "2". DROP TABLE alter_compress_params_schema.uncompress_to_compresstype_3; -- set compressed options of column table CREATE TABLE alter_compress_params_schema.alter_column_table_compressed_options (id int, value varchar) WITH (ORIENTATION = column); diff --git a/src/test/regress/expected/row_compression/unsupported_feature.out b/src/test/regress/expected/row_compression/unsupported_feature.out index 7eba25532..9658f08ef 100644 --- a/src/test/regress/expected/row_compression/unsupported_feature.out +++ b/src/test/regress/expected/row_compression/unsupported_feature.out @@ -2,7 +2,7 @@ create schema unsupported_feature; -- unspport compressType: 4 CREATE TABLE unsupported_feature.compressed_table_1024(id int) WITH(compresstype=4, compress_chunk_size=1024); ERROR: value 4 out of bounds for option "compresstype" -DETAIL: Valid values are between "0" and "3". +DETAIL: Valid values are between "0" and "2". -- unspport compress_chunk_size: 2000 CREATE TABLE unsupported_feature.compressed_table_1024(id int) WITH(compresstype=2, compress_chunk_size=2000); ERROR: invalid compress_chunk_size 2000, must be one of 512, 1024, 2048 or 4096 for compressed_table_1024 From c477ed5163fefa4fc116c1d0991feb824c502454 Mon Sep 17 00:00:00 2001 From: duzhuolin Date: Fri, 28 Jul 2023 15:04:20 +0800 Subject: [PATCH 084/304] =?UTF-8?q?=E4=BF=AE=E6=94=B9spi=5Fpriv=E3=80=81ex?= =?UTF-8?q?ecRemote=E5=91=8A=E8=AD=A6=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/pgxc_single/pool/execRemote.cpp | 4 ++-- src/include/executor/spi_priv.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/common/backend/pgxc_single/pool/execRemote.cpp b/src/common/backend/pgxc_single/pool/execRemote.cpp index cb6d6577f..3568ab9db 100755 --- a/src/common/backend/pgxc_single/pool/execRemote.cpp +++ b/src/common/backend/pgxc_single/pool/execRemote.cpp @@ -5299,11 +5299,11 @@ void do_query_for_first_tuple(RemoteQueryState* node, bool vectorized, int regul pfree_ext(node->cursor_connections); if (!node->need_error_check) { - int error_code; + int error_code = 0; char* error_msg = getSocketError(&error_code); ereport(ERROR, - (errcode(error_code), errmsg("Failed to read response from Datanodes Detail: %s\n", error_msg))); + (errcode(error_code), errmsg("Failed to read response from Datanodes Detail: %s\n", error_msg == NULL ? "null" : error_msg))); } else { node->need_error_check = false; pgxc_node_report_error(node); diff --git a/src/include/executor/spi_priv.h b/src/include/executor/spi_priv.h index 5b7e9ef86..e7f99af22 100644 --- a/src/include/executor/spi_priv.h +++ b/src/include/executor/spi_priv.h @@ -110,7 +110,7 @@ inline void spi_stack_record_log(const char* action, const char* filename, int l ereport(DEBUG3, (errmodule(MOD_SPI), errcode(ERRCODE_LOG), errmsg("SPISTACK(Action:%s, Location %s,%d, Funcname:%s): cur spiconnected:%d, cur spi:%d, query string:%s", action, filename, lineno, funcname, u_sess->SPI_cxt._connected, u_sess->SPI_cxt._curid, - query != NULL ? maskPassword(query) : NULL))); + query != NULL ? maskPassword(query) : "null"))); } } From 7832090f9b2c5f92ba3ff15ab15b510afafde704 Mon Sep 17 00:00:00 2001 From: Julong-Li <584147810@qq.com> Date: Thu, 27 Jul 2023 16:14:24 +0800 Subject: [PATCH 085/304] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E5=8C=BF=E5=90=8D?= =?UTF-8?q?=E5=9D=97=E6=89=A7=E8=A1=8C=E6=95=B0=E6=8D=AE=E5=BA=93=E5=AE=95?= =?UTF-8?q?=E6=9C=BA=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/pl/plpgsql/src/gram.y | 1 + src/test/regress/expected/test_plsql_core.out | 35 +++++++++++++++++++ src/test/regress/parallel_schedule0 | 4 +-- src/test/regress/parallel_schedule0A | 4 +-- src/test/regress/sql/test_plsql_core.sql | 12 +++++++ 5 files changed, 52 insertions(+), 4 deletions(-) create mode 100644 src/test/regress/expected/test_plsql_core.out create mode 100644 src/test/regress/sql/test_plsql_core.sql diff --git a/src/common/pl/plpgsql/src/gram.y b/src/common/pl/plpgsql/src/gram.y index fe3217956..f87a40837 100755 --- a/src/common/pl/plpgsql/src/gram.y +++ b/src/common/pl/plpgsql/src/gram.y @@ -10064,6 +10064,7 @@ make_execsql_stmt(int firsttoken, int location) PLpgSQL_nsitem* ns = plpgsql_ns_lookup(plpgsql_ns_top(), false, yylval.word.ident, NULL, NULL, NULL); if (ns == NULL) { yyerror("insert an nonexistent variable."); + continue; } PLpgSQL_datum* datum = u_sess->plsql_cxt.curr_compile_context->plpgsql_Datums[ns->itemno]; diff --git a/src/test/regress/expected/test_plsql_core.out b/src/test/regress/expected/test_plsql_core.out new file mode 100644 index 000000000..a91dd2afb --- /dev/null +++ b/src/test/regress/expected/test_plsql_core.out @@ -0,0 +1,35 @@ +set plsql_show_all_error to on; +begin +forall i in 1 .. v1.count save exceptions insert into tb values v1(i); +end; +/ +NOTICE: insert an nonexistent variable. at or near "v1" +LINE 2: ... 1 .. v1.count save exceptions insert into tb values v1(i); + ^ +QUERY: DECLARE +BEGIN forall i in 1 .. v1.count save exceptions insert into tb values v1(i); +end +ERROR: unsupported insert into table from non record type. at or near "i" +LINE 2: ... 1 .. v1.count save exceptions insert into tb values v1(i); + ^ +QUERY: DECLARE +BEGIN forall i in 1 .. v1.count save exceptions insert into tb values v1(i); +end +create procedure test_plsql_core() +is +begin +forall i in 1 .. v1.count save exceptions insert into tb values v1(i); +end; +/ +NOTICE: insert an nonexistent variable. at or near "v1" +LINE 2: ...n 1 .. v1.count save exceptions insert into tb values v1(i); + ^ +QUERY: DECLARE begin +forall i in 1 .. v1.count save exceptions insert into tb values v1(i); +end +ERROR: unsupported insert into table from non record type. at or near "i" +LINE 2: ...n 1 .. v1.count save exceptions insert into tb values v1(i); + ^ +QUERY: DECLARE begin +forall i in 1 .. v1.count save exceptions insert into tb values v1(i); +end diff --git a/src/test/regress/parallel_schedule0 b/src/test/regress/parallel_schedule0 index f7f9846b4..8a1a799b5 100644 --- a/src/test/regress/parallel_schedule0 +++ b/src/test/regress/parallel_schedule0 @@ -1056,8 +1056,8 @@ test: describe_index_with_tablespace test: sytcomp_del_upt4orderby test: aioptimizer test: aioptimizer_small -test: pgfincore -test: rename_table +test: pgfincore +test: rename_table test_plsql_core # debug instrument test: test_debug5 diff --git a/src/test/regress/parallel_schedule0A b/src/test/regress/parallel_schedule0A index 7582f49de..7ef668d0b 100644 --- a/src/test/regress/parallel_schedule0A +++ b/src/test/regress/parallel_schedule0A @@ -57,7 +57,7 @@ test: out_param_func test: gs_dump_tableconstraint # test AI4DB -test: plpgsql_override_out +test: plpgsql_override_out test_plsql_core test: plpgsql_sql_with_proc_keyword test: plsql_show_all_error b_pg_plsql_show_all_error test: pldeveloper_gs_source @@ -460,4 +460,4 @@ test: alter_table_modify alter_table_modify_ustore alter_table_modify_ltt alter_ # test for empty string in A format database test: accept_empty_str not_accept_empty_str pg_empty_str accept_empty_copy not_accept_empty_copy -#test: with \ No newline at end of file +#test: with diff --git a/src/test/regress/sql/test_plsql_core.sql b/src/test/regress/sql/test_plsql_core.sql new file mode 100644 index 000000000..a5068c40f --- /dev/null +++ b/src/test/regress/sql/test_plsql_core.sql @@ -0,0 +1,12 @@ +set plsql_show_all_error to on; +begin +forall i in 1 .. v1.count save exceptions insert into tb values v1(i); +end; +/ + +create procedure test_plsql_core() +is +begin +forall i in 1 .. v1.count save exceptions insert into tb values v1(i); +end; +/ From 1505986cd145420b6b6183d24c76156576b6cbed Mon Sep 17 00:00:00 2001 From: duzhuolin Date: Fri, 28 Jul 2023 15:08:32 +0800 Subject: [PATCH 086/304] =?UTF-8?q?=E4=BF=AE=E6=94=B9pooler=5Fmaxium=5Fidl?= =?UTF-8?q?e=5Ftime=E5=8F=82=E6=95=B0=E6=A0=A1=E9=AA=8C=E5=87=BD=E6=95=B0?= =?UTF-8?q?=E7=9A=84=E6=8A=A5=E9=94=99=E6=97=A5=E5=BF=97=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/utils/misc/guc/guc_network.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/backend/utils/misc/guc/guc_network.cpp b/src/common/backend/utils/misc/guc/guc_network.cpp index 2d3475e89..14104f334 100755 --- a/src/common/backend/utils/misc/guc/guc_network.cpp +++ b/src/common/backend/utils/misc/guc/guc_network.cpp @@ -1183,7 +1183,7 @@ static bool check_ssl(bool* newval, void** extra, GucSource source) static bool check_pooler_maximum_idle_time(int* newval, void** extra, GucSource source) { if (*newval < 0) { - ereport(ERROR, (errmsg("GaussDB can't support idle time less than 0 or more than %d seconds.", INT_MAX))); + GUC_check_errmsg("GaussDB can't support idle time less than 0 or more than %d seconds.", INT_MAX); return false; } From eada31bc1320088600bb0963b43397dbe0c21b91 Mon Sep 17 00:00:00 2001 From: Luan-233 <2533556772@qq.com> Date: Fri, 28 Jul 2023 16:42:01 +0800 Subject: [PATCH 087/304] fix a bug when modify the parameter audit_data_format, if length is more than MAX_LENGTH, the errorcode is not the expected one. --- src/common/backend/utils/misc/guc.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/common/backend/utils/misc/guc.cpp b/src/common/backend/utils/misc/guc.cpp index d5f45e920..08c83557f 100755 --- a/src/common/backend/utils/misc/guc.cpp +++ b/src/common/backend/utils/misc/guc.cpp @@ -8580,6 +8580,11 @@ static void replace_config_value(char** optlines, char* name, char* value, confi rc = snprintf_s(newline, MAX_PARAM_LEN, MAX_PARAM_LEN - 1, "%s = %s\n", name, value); break; case PGC_STRING: + if (strlen(name) + strlen(value) + 6 >= MAX_PARAM_LEN) { + pfree(newline); + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("too long for GUC parameter."))); + } rc = snprintf_s(newline, MAX_PARAM_LEN, MAX_PARAM_LEN - 1, "%s = '%s'\n", name, value); break; } From 47c74683764c51eb898000cfd12d5c4a310d9e2f Mon Sep 17 00:00:00 2001 From: Luan-233 <2533556772@qq.com> Date: Fri, 28 Jul 2023 17:23:09 +0800 Subject: [PATCH 088/304] fix a bug when modify the parameter audit_data_format, if length is more than MAX_LENGTH, the errorcode is not the expected one. --- src/common/backend/utils/misc/guc.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/common/backend/utils/misc/guc.cpp b/src/common/backend/utils/misc/guc.cpp index 08c83557f..4ec7f9484 100755 --- a/src/common/backend/utils/misc/guc.cpp +++ b/src/common/backend/utils/misc/guc.cpp @@ -8582,8 +8582,7 @@ static void replace_config_value(char** optlines, char* name, char* value, confi case PGC_STRING: if (strlen(name) + strlen(value) + 6 >= MAX_PARAM_LEN) { pfree(newline); - ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("too long for GUC parameter."))); + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("the length of GUC string type value exceed 1024."))); } rc = snprintf_s(newline, MAX_PARAM_LEN, MAX_PARAM_LEN - 1, "%s = '%s'\n", name, value); break; From 713571a7e03f93a2e1f78cc29de3e14cbaaad889 Mon Sep 17 00:00:00 2001 From: hwhbj Date: Sat, 29 Jul 2023 10:12:26 +0800 Subject: [PATCH 089/304] add build assessment plugin --- build/script/utils/make_compile.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/build/script/utils/make_compile.sh b/build/script/utils/make_compile.sh index 8b10a9c7b..d9e9650f2 100644 --- a/build/script/utils/make_compile.sh +++ b/build/script/utils/make_compile.sh @@ -219,6 +219,12 @@ function install_gaussdb() make install -sj >> "$LOG_FILE" 2>&1 echo "End make install MPPDB" >> "$LOG_FILE" 2>&1 + ASSESSMENT_DIR=$ROOT_DIR/contrib/assessment + if [ -d $ASSESSMENT_DIR ]; then + cd $ASSESSMENT_DIR + make install >> "$LOG_FILE" 2>&1 + echo "End make install assessment" >> "$LOG_FILE" 2>&1 + fi cd "$ROOT_DIR" if [ "${make_check}" = 'on' ]; then From a64cd6745f55a41306230d45bcd93205f741a157 Mon Sep 17 00:00:00 2001 From: hwhbj Date: Mon, 31 Jul 2023 09:57:39 +0800 Subject: [PATCH 090/304] =?UTF-8?q?=E4=BD=BF=E7=94=A8sh=20build.sh=20-pkg?= =?UTF-8?q?=E6=89=93=E5=8C=85=E6=B7=BB=E5=8A=A0assessment=E6=8F=92?= =?UTF-8?q?=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build/script/aarch64_opengauss_list | 4 ++++ build/script/x86_64_opengauss_list | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/build/script/aarch64_opengauss_list b/build/script/aarch64_opengauss_list index bf5741dbd..e776d1ee9 100644 --- a/build/script/aarch64_opengauss_list +++ b/build/script/aarch64_opengauss_list @@ -47,6 +47,7 @@ ./bin/gs_plan_simulator.sh ./bin/pg_xlogdump ./bin/pagehack +./bin/assessment_database ./etc/kerberos/kadm5.acl ./etc/kerberos/kdc.conf ./etc/kerberos/krb5.conf @@ -78,6 +79,8 @@ ./share/postgresql/extension/dolphin--1.1--2.0.sql ./share/postgresql/extension/dolphin--2.0--1.1.sql ./share/postgresql/extension/openGauss_expr_dolphin.ir +./share/postgresql/extension/assessment--1.0.sql +./share/postgresql/extension/assessment.control ./share/postgresql/extension/file_fdw--1.0.sql ./share/postgresql/extension/plpgsql.control ./share/postgresql/extension/dist_fdw.control @@ -773,6 +776,7 @@ ./lib/postgresql/postgres_fdw.so ./lib/postgresql/dblink.so ./lib/postgresql/pgoutput.so +./lib/postgresql/assessment.so ./lib/libpljava.so ./lib/libpq.a ./lib/libpq.so diff --git a/build/script/x86_64_opengauss_list b/build/script/x86_64_opengauss_list index ae72548b1..5f2d0b1a5 100644 --- a/build/script/x86_64_opengauss_list +++ b/build/script/x86_64_opengauss_list @@ -47,6 +47,7 @@ ./bin/gs_plan_simulator.sh ./bin/pg_xlogdump ./bin/pagehack +./bin/assessment_database ./etc/kerberos/kadm5.acl ./etc/kerberos/kdc.conf ./etc/kerberos/krb5.conf @@ -78,6 +79,8 @@ ./share/postgresql/extension/dolphin--1.1--2.0.sql ./share/postgresql/extension/dolphin--2.0--1.1.sql ./share/postgresql/extension/openGauss_expr_dolphin.ir +./share/postgresql/extension/assessment--1.0.sql +./share/postgresql/extension/assessment.control ./share/postgresql/extension/file_fdw--1.0.sql ./share/postgresql/extension/plpgsql.control ./share/postgresql/extension/dist_fdw.control @@ -773,6 +776,7 @@ ./lib/postgresql/postgres_fdw.so ./lib/postgresql/dblink.so ./lib/postgresql/pgoutput.so +./lib/postgresql/assessment.so ./lib/libpljava.so ./lib/libpq.a ./lib/libpq.so From 967d869f667cc2fa9659868c3a93d204baeaac0c Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Mon, 31 Jul 2023 20:21:05 +0800 Subject: [PATCH 091/304] =?UTF-8?q?=E5=8F=8C=E9=9B=86=E7=BE=A4build?= =?UTF-8?q?=E9=80=82=E9=85=8Ddss=E8=B7=B3=E8=BF=87=E7=9B=AE=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/pg_ctl/pg_build.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bin/pg_ctl/pg_build.cpp b/src/bin/pg_ctl/pg_build.cpp index 3724f141c..f06d36aa3 100755 --- a/src/bin/pg_ctl/pg_build.cpp +++ b/src/bin/pg_ctl/pg_build.cpp @@ -1931,7 +1931,7 @@ int fsync_fname(const char *fname, bool isdir) */ fd = open(fname, flags, 0); if (fd < 0) { - if (errno == EACCES || (isdir && errno == EISDIR)) + if (errno == EACCES || (isdir && (errno == EISDIR || errno == ERR_DSS_FILE_TYPE_MISMATCH))) return 0; pg_log(PG_WARNING, _("could not open file \"%s\": %s\n"), fname, strerror(errno)); return -1; From 8bb32f7caf8d8b0ebce1d693d23977b5ffbb4262 Mon Sep 17 00:00:00 2001 From: chendong76 <1209756284@qq.com> Date: Tue, 1 Aug 2023 10:33:31 +0800 Subject: [PATCH 092/304] =?UTF-8?q?=E3=80=90bugfix=E3=80=91=E8=A7=A3?= =?UTF-8?q?=E5=86=B3=E6=8C=89=E9=9C=80=E5=9B=9E=E6=94=BE=E9=83=A8=E5=88=86?= =?UTF-8?q?=E9=97=AE=E9=A2=98=EF=BC=9A1.=20=E6=8C=89=E9=9C=80=E5=9B=9E?= =?UTF-8?q?=E6=94=BE=E6=88=90=E5=8A=9F=E5=90=8E=EF=BC=8C=E5=86=8D=E6=AC=A1?= =?UTF-8?q?=E8=BF=9B=E5=85=A5=E5=9B=9E=E6=94=BE=E5=90=8E=EF=BC=8C=E5=9B=9E?= =?UTF-8?q?=E6=94=BE=E6=96=B9=E5=BC=8F=E9=80=89=E6=8B=A9=E9=94=99=E8=AF=AF?= =?UTF-8?q?;=202.=20=E6=8C=89=E9=9C=80=E5=9B=9E=E6=94=BE=E5=A4=B1=E8=B4=A5?= =?UTF-8?q?=EF=BC=8C=E5=85=B6=E5=AE=83=E8=8A=82=E7=82=B9=E5=9B=9E=E6=94=BE?= =?UTF-8?q?=E6=88=90=E5=8A=9F=E5=90=8E=EF=BC=8C=E5=9B=9E=E6=94=BE=E5=A4=B1?= =?UTF-8?q?=E8=B4=A5=E8=8A=82=E7=82=B9=E6=97=A0=E6=B3=95=E6=8B=89=E8=B5=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/ddes/adapter/ss_dms_bufmgr.cpp | 2 +- src/gausskernel/process/postmaster/postmaster.cpp | 2 +- .../storage/access/transam/extreme_rto_redo_api.cpp | 5 +++++ src/gausskernel/storage/access/transam/xlog.cpp | 13 +++++++++---- src/include/access/extreme_rto_redo_api.h | 1 + 5 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/gausskernel/ddes/adapter/ss_dms_bufmgr.cpp b/src/gausskernel/ddes/adapter/ss_dms_bufmgr.cpp index 1cc615efe..ba644e984 100644 --- a/src/gausskernel/ddes/adapter/ss_dms_bufmgr.cpp +++ b/src/gausskernel/ddes/adapter/ss_dms_bufmgr.cpp @@ -521,7 +521,7 @@ bool SSOndemandRequestPrimaryRedo(BufferTag tag) (unsigned int)sizeof(BufferTag), &redo_status) != DMS_SUCCESS) { ereport(LOG, (errmodule(MOD_DMS), - errmsg("[on-demand] request primary node redo page failed, page id [%d/%d/%d/%d/%d %d-%d], " + errmsg("[On-demand] request primary node redo page failed, page id [%d/%d/%d/%d/%d %d-%d], " "redo status %d", tag.rnode.spcNode, tag.rnode.dbNode, tag.rnode.relNode, (int)tag.rnode.bucketNode, (int)tag.rnode.opt, tag.forkNum, tag.blockNum, redo_status))); return false; diff --git a/src/gausskernel/process/postmaster/postmaster.cpp b/src/gausskernel/process/postmaster/postmaster.cpp index c7e1ec2e4..1b60ba630 100644 --- a/src/gausskernel/process/postmaster/postmaster.cpp +++ b/src/gausskernel/process/postmaster/postmaster.cpp @@ -3030,7 +3030,7 @@ int PostmasterMain(int argc, char* argv[]) if (SS_OFFICIAL_RECOVERY_NODE && SS_CLUSTER_ONDEMAND_NOT_NORAML) { ereport(FATAL, (errmsg( - "[on-demand] node%d is last primary node, do not allow join cluster until on-demand recovery done", + "[On-demand] node%d is last primary node, do not allow join cluster until on-demand recovery done", g_instance.attr.attr_storage.dms_attr.instance_id))); } } diff --git a/src/gausskernel/storage/access/transam/extreme_rto_redo_api.cpp b/src/gausskernel/storage/access/transam/extreme_rto_redo_api.cpp index bc5ca2582..f8c7eddea 100644 --- a/src/gausskernel/storage/access/transam/extreme_rto_redo_api.cpp +++ b/src/gausskernel/storage/access/transam/extreme_rto_redo_api.cpp @@ -141,6 +141,11 @@ static const f_extreme_rto_redo extreme_rto_redosw[] = { }, }; +void SetExtremeRtoMode() +{ + g_extreme_rto_type = DEFAULT_EXTREME_RTO; +} + void SetOndemandExtremeRtoMode() { g_extreme_rto_type = ONDEMAND_EXTREME_RTO; diff --git a/src/gausskernel/storage/access/transam/xlog.cpp b/src/gausskernel/storage/access/transam/xlog.cpp index c5427d7d2..6a01a780b 100755 --- a/src/gausskernel/storage/access/transam/xlog.cpp +++ b/src/gausskernel/storage/access/transam/xlog.cpp @@ -8808,11 +8808,11 @@ void StartupXLOG(void) SSReadControlFile(REFORM_CTRL_PAGE); if (SS_CLUSTER_ONDEMAND_NOT_NORAML && SS_PRIMARY_MODE) { if (SS_STANDBY_PROMOTING) { - ereport(FATAL, (errmsg("Do not allow switchover if on-demand recovery is not finish"))); + ereport(FATAL, (errmsg("[On-demand] Do not allow switchover if ondemand recovery is not finish"))); } Assert(g_instance.dms_cxt.SSReformerControl.recoveryInstId != INVALID_INSTANCEID); src_id = g_instance.dms_cxt.SSReformerControl.recoveryInstId; - ereport(LOG, (errmsg("[on-demand]: On-demand recovery do not finish in last reform, " + ereport(LOG, (errmsg("[On-demand]: Ondemand recovery do not finish in last reform, " "reading control file of original primary:%d", src_id))); SSOndemandRecoveryExitNormal = false; } else { @@ -9493,6 +9493,8 @@ void StartupXLOG(void) SetOndemandExtremeRtoMode(); ereport(LOG, (errmsg("[On-demand] replayed in extreme rto ondemand recovery mode"))); } else { + g_instance.dms_cxt.SSRecoveryInfo.in_ondemand_recovery = false; + SetExtremeRtoMode(); ereport(LOG, (errmsg("[On-demand] do not allow replay in ondemand recovery if last ondemand recovery " "crash, replayed in extreme rto recovery mode"))); } @@ -10517,7 +10519,7 @@ void StartupXLOG(void) if (SS_IN_ONDEMAND_RECOVERY) { /* We wait at here */ - ereport(LOG, (errmsg("[SS] On-demand redo, nextXid: " XID_FMT ", startupMaxXid: " XID_FMT + ereport(LOG, (errmsg("[On-demand] ondemand redo, nextXid: " XID_FMT ", startupMaxXid: " XID_FMT ", recentLocalXmin: " XID_FMT ", recentGlobalXmin: %lu, PendingPreparedXacts: %d" ", NextCommitSeqNo: %lu, cutoff_csn_min: %lu.", NextXidAfterReovery, t_thrd.xact_cxt.ShmemVariableCache->startupMaxXid, @@ -10538,7 +10540,7 @@ void StartupXLOG(void) LWLockRelease(ControlFileLock); SSRecheckBufferPool(); ereport(LOG, (errmodule(MOD_DMS), - errmsg("[SS][on demand recovery] finished full checkpoint and update control file"))); + errmsg("[On-demand] finished full checkpoint and update control file"))); NotifyGscRecoveryFinished(); if (ENABLE_INCRE_CKPT) { @@ -11568,6 +11570,9 @@ void CreateCheckPoint(int flags) } else if (g_instance.dms_cxt.SSRecoveryInfo.failover_ckpt_status == NOT_ALLOW_CKPT) { ereport(LOG, (errmodule(MOD_DMS), errmsg("[SS failover] do not do CreateCheckpoint during failover"))); return; + } else if (SS_IN_ONDEMAND_RECOVERY && !SS_ONDEMAND_REDO_DONE) { + /* do not allow ckpt in ondemand recovery if xlog do not redo done, for valid ckpt loc in control file */ + return; } /* CHECKPOINT_IS_SHUTDOWN CHECKPOINT_END_OF_RECOVERY CHECKPOINT_FORCE shuld do full checkpoint */ diff --git a/src/include/access/extreme_rto_redo_api.h b/src/include/access/extreme_rto_redo_api.h index 652d13c15..ef86785bf 100644 --- a/src/include/access/extreme_rto_redo_api.h +++ b/src/include/access/extreme_rto_redo_api.h @@ -36,6 +36,7 @@ typedef enum { extern ExtremeRtoRedoType g_extreme_rto_type; +void SetExtremeRtoMode(); void SetOndemandExtremeRtoMode(); void ExtremeWaitAllReplayWorkerIdle(); void ExtremeDispatchCleanInvalidPageMarkToAllRedoWorker(RepairFileKey key); From 8169f6fcb6e903cef95f00f54a600091091d1d54 Mon Sep 17 00:00:00 2001 From: zhangao_za <18829237393@163.com> Date: Tue, 1 Aug 2023 10:35:00 +0800 Subject: [PATCH 093/304] =?UTF-8?q?=E9=80=82=E9=85=8D=E5=88=9D=E5=A7=8B?= =?UTF-8?q?=E5=8C=96=E6=97=B6ENABLE=5FDMS=E6=9C=AA=E7=94=9F=E6=95=88?= =?UTF-8?q?=E7=9A=84=E5=9C=BA=E6=99=AF=EF=BC=8Ctoast=E8=A1=A8=E4=B8=8D?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0segment=3Don=E9=80=89=E9=A1=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/catalog/heap.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/backend/catalog/heap.cpp b/src/common/backend/catalog/heap.cpp index b1a2be1f9..905451180 100644 --- a/src/common/backend/catalog/heap.cpp +++ b/src/common/backend/catalog/heap.cpp @@ -2627,7 +2627,7 @@ Oid heap_create_with_catalog(const char *relname, Oid relnamespace, Oid reltable /* store tables in segment storage as all possible while initdb */ if (relpersistence == RELPERSISTENCE_PERMANENT && (relkind != RELKIND_SEQUENCE && relkind != RELKIND_LARGE_SEQUENCE && - (!ENABLE_DMS || relkind != RELKIND_TOASTVALUE))) { + relkind != RELKIND_TOASTVALUE)) { storage_type = SEGMENT_PAGE; reloptions = AddSegmentOption(reloptions); } From ba1430fdc2a6f969b518b4ac2a8925b57afbfa0c Mon Sep 17 00:00:00 2001 From: duzhuolin Date: Tue, 1 Aug 2023 10:37:03 +0800 Subject: [PATCH 094/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=86=85=E5=AD=98?= =?UTF-8?q?=E6=B3=84=E9=9C=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/catalog/gs_package.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/common/backend/catalog/gs_package.cpp b/src/common/backend/catalog/gs_package.cpp index 7186bd80c..c6ff29542 100755 --- a/src/common/backend/catalog/gs_package.cpp +++ b/src/common/backend/catalog/gs_package.cpp @@ -1387,6 +1387,8 @@ static void RestorePkgValuesByPkgState(PLpgSQL_package* targetPkg, PackageRuntim if (fromVar->tableOfIndex != NULL) { MemoryContext temp = MemoryContextSwitchTo(targetVar->pkg->pkg_cxt); + hash_destroy(targetVar->tableOfIndex); + targetVar->tableOfIndex = NULL; targetVar->tableOfIndex = copyTableOfIndex(fromVar->tableOfIndex); MemoryContextSwitchTo(temp); } else if (fromVar->isnull) { From dcce3951bce62e48d03e3ec98af1221878d8b750 Mon Sep 17 00:00:00 2001 From: zhaojun Date: Tue, 1 Aug 2023 14:44:45 +0800 Subject: [PATCH 095/304] add end_command func for dolphin proto --- .../process/postmaster/postmaster.cpp | 1 + src/gausskernel/process/tcop/dest.cpp | 19 ++++++++++++------- src/include/libpq/libpq-be.h | 1 + 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/gausskernel/process/postmaster/postmaster.cpp b/src/gausskernel/process/postmaster/postmaster.cpp index 5fab30e12..1a3654d61 100644 --- a/src/gausskernel/process/postmaster/postmaster.cpp +++ b/src/gausskernel/process/postmaster/postmaster.cpp @@ -565,6 +565,7 @@ ProtocolExtensionConfig default_protocol_config = { pq_comm_reset, ReadyForQuery, SocketBackend, + NULL, printtup_create_DR, /* use libpq defaults for printtup*() */ NULL, NULL, diff --git a/src/gausskernel/process/tcop/dest.cpp b/src/gausskernel/process/tcop/dest.cpp index 977871fe2..c6366759c 100644 --- a/src/gausskernel/process/tcop/dest.cpp +++ b/src/gausskernel/process/tcop/dest.cpp @@ -187,14 +187,19 @@ void EndCommand(const char* commandTag, CommandDest dest) case DestTupleRedistribute: case DestBatchBroadCast: case DestBatchLocalBroadCast: - case DestBatchRedistribute: - /* - * We assume the commandTag is plain ASCII and therefore requires - * no encoding conversion. - */ - pq_putmessage('C', commandTag, strlen(commandTag) + 1); + case DestBatchRedistribute: { + Port *MyPort = u_sess->proc_cxt.MyProcPort; + if (MyPort && MyPort->protocol_config && MyPort->protocol_config->fn_end_command) { + MyPort->protocol_config->fn_end_command(commandTag); + } else { + /* + * We assume the commandTag is plain ASCII and therefore requires + * no encoding conversion. + */ + pq_putmessage('C', commandTag, strlen(commandTag) + 1); + } break; - + } case DestNone: case DestDebug: case DestSPI: diff --git a/src/include/libpq/libpq-be.h b/src/include/libpq/libpq-be.h index 96b920191..591f78247 100644 --- a/src/include/libpq/libpq-be.h +++ b/src/include/libpq/libpq-be.h @@ -104,6 +104,7 @@ typedef struct ProtocolExtensionConfig { void (*fn_comm_reset)(void); void (*fn_send_ready_for_query)(CommandDest dest); int (*fn_read_command)(StringInfo inBuf); + void (*fn_end_command)(const char *completionTag); DestReceiver* (*fn_printtup_create_DR)(CommandDest dest); void (*fn_set_DR_params)(DestReceiver* self, List* target_list); int (*fn_process_command)(StringInfo inBuf); From 1108ae26e5c840383f3ede5f35dabcb67133620b Mon Sep 17 00:00:00 2001 From: totaj Date: Tue, 1 Aug 2023 14:33:19 +0800 Subject: [PATCH 096/304] Fix select @@ bool parameter bug. --- src/common/backend/parser/parse_coerce.cpp | 2 +- .../regress/input/set_system_variables_test.source | 3 ++- .../regress/output/set_system_variables_test.source | 12 +++++++++--- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/common/backend/parser/parse_coerce.cpp b/src/common/backend/parser/parse_coerce.cpp index af255e1db..02fb56e2d 100644 --- a/src/common/backend/parser/parse_coerce.cpp +++ b/src/common/backend/parser/parse_coerce.cpp @@ -3137,7 +3137,7 @@ Const* setValueToConstExpr(SetVariableExpr* set) case PGC_BOOL: { bool variable_bool = false; - if (strcmp(variable_str,"true") || strcmp(variable_str,"on")) { + if (strcmp(variable_str, "true") == 0 || strcmp(variable_str,"on") == 0) { variable_bool = true; } val = BoolGetDatum(variable_bool); diff --git a/src/test/regress/input/set_system_variables_test.source b/src/test/regress/input/set_system_variables_test.source index e05b69ac9..072996398 100644 --- a/src/test/regress/input/set_system_variables_test.source +++ b/src/test/regress/input/set_system_variables_test.source @@ -110,6 +110,7 @@ set @@session.enable_broadcast = on; set @@session.enable_broadcast = default; set @@enable_broadcast = (1 = 1); set @@enable_broadcast = (true = true); +select @@enable_broadcast; set @@enable_broadcast = (true = false); select @@enable_broadcast; @@ -298,4 +299,4 @@ drop database if exists test_set; \! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1 -c "enable_set_variable_b_format=off" >/dev/null 2>&1 \! sleep 1 -show enable_set_variable_b_format; \ No newline at end of file +show enable_set_variable_b_format; diff --git a/src/test/regress/output/set_system_variables_test.source b/src/test/regress/output/set_system_variables_test.source index c08cff1fd..145d12f84 100644 --- a/src/test/regress/output/set_system_variables_test.source +++ b/src/test/regress/output/set_system_variables_test.source @@ -237,23 +237,29 @@ set @@session.enable_broadcast = on; set @@session.enable_broadcast = default; set @@enable_broadcast = (1 = 1); set @@enable_broadcast = (true = true); -set @@enable_broadcast = (true = false); select @@enable_broadcast; ?column? ---------- t (1 row) +set @@enable_broadcast = (true = false); +select @@enable_broadcast; + ?column? +---------- + f +(1 row) + select @@session.enable_broadcast; ?column? ---------- - t + f (1 row) select @@session.enable_broadcast = 'true'; ?column? ---------- - t + f (1 row) set global most_available_sync = t; From e5311bd6a1eaa95204ea156cd9cef66497bcea3e Mon Sep 17 00:00:00 2001 From: dongning12 Date: Tue, 1 Aug 2023 17:06:47 +0800 Subject: [PATCH 097/304] =?UTF-8?q?=E3=80=90=E8=B5=84=E6=BA=90=E6=B1=A0?= =?UTF-8?q?=E5=8C=96=E3=80=91DMS=E4=BE=A7=E6=B6=88=E6=81=AF=E8=B6=85?= =?UTF-8?q?=E6=97=B6=E6=97=B6=E9=97=B4=E9=BB=98=E8=AE=A410s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/ddes/adapter/ss_init.cpp | 1 + src/include/ddes/dms/ss_common_attr.h | 2 ++ 2 files changed, 3 insertions(+) diff --git a/src/gausskernel/ddes/adapter/ss_init.cpp b/src/gausskernel/ddes/adapter/ss_init.cpp index 54b437919..a4f862cbc 100644 --- a/src/gausskernel/ddes/adapter/ss_init.cpp +++ b/src/gausskernel/ddes/adapter/ss_init.cpp @@ -377,6 +377,7 @@ static void setDMSProfile(dms_profile_t* profile) profile->enable_reform = (unsigned char)dms_attr->enable_reform; profile->load_balance_mode = 1; /* primary-standby */ profile->parallel_thread_num = dms_attr->parallel_thread_num; + profile->max_wait_time = DMS_MSG_MAX_WAIT_TIME; if (dms_attr->enable_ssl && g_instance.attr.attr_security.EnableSSL) { InitDmsSSL(); diff --git a/src/include/ddes/dms/ss_common_attr.h b/src/include/ddes/dms/ss_common_attr.h index c04d0a63a..fca52f775 100644 --- a/src/include/ddes/dms/ss_common_attr.h +++ b/src/include/ddes/dms/ss_common_attr.h @@ -186,6 +186,8 @@ #define SS_ACQUIRE_LOCK_DO_NOT_WAIT 0 #define SS_ACQUIRE_LOCK_RETRY_INTERVAL (50) // 50ms +#define DMS_MSG_MAX_WAIT_TIME (10 * 1000) // 10s + typedef enum SSBroadcastOp { BCAST_GET_XMIN = 0, BCAST_CANCEL_TRX_FOR_SWITCHOVER, From acf1fe375c587e646392e4c998f27740a861ab92 Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Wed, 2 Aug 2023 15:51:46 +0800 Subject: [PATCH 098/304] =?UTF-8?q?=E5=8F=8C=E9=9B=86=E7=BE=A4build?= =?UTF-8?q?=E5=88=A0=E9=99=A4=E5=A4=8D=E5=88=B6=E6=A7=BD=E8=BD=AF=E8=BF=9E?= =?UTF-8?q?=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/pg_ctl/pg_build.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/bin/pg_ctl/pg_build.cpp b/src/bin/pg_ctl/pg_build.cpp index f06d36aa3..a84b1c2a3 100755 --- a/src/bin/pg_ctl/pg_build.cpp +++ b/src/bin/pg_ctl/pg_build.cpp @@ -1417,9 +1417,12 @@ static void DeleteSubDataDir(const char* dirname) de_slot->d_name); securec_check_ss_c(nRet, "", ""); if (!rmtree(fullpath, true)) { - pg_log(PG_WARNING, _("failed to remove dir %s,errno=%d.\n"), fullpath, errno); - (void)closedir(dir); - exit(1); + /* enable dss, something in pg_replslot may be a link */ + if (unlink(fullpath) != 0) { + pg_log(PG_WARNING, _("failed to remove dir %s,errno=%d.\n"), fullpath, errno); + (void)closedir(dir); + exit(1); + } } } (void)closedir(dir_slot); From e8c0b6674fb9a870349ec1132d1029dbd8cd270f Mon Sep 17 00:00:00 2001 From: li-judong Date: Wed, 2 Aug 2023 20:28:02 +0800 Subject: [PATCH 099/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E9=9A=90=E5=BC=8F?= =?UTF-8?q?=E8=BD=AC=E6=8D=A2=E4=B8=A2=E5=A4=B1=E5=AD=97=E7=AC=A6=E5=BA=8F?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/parser/parse_coerce.cpp | 25 ++++--- .../regress/expected/charset_utf8mb4_b_db.out | 67 +++++++++++++++++-- src/test/regress/sql/charset_utf8mb4_b_db.sql | 38 ++++++++++- 3 files changed, 112 insertions(+), 18 deletions(-) diff --git a/src/common/backend/parser/parse_coerce.cpp b/src/common/backend/parser/parse_coerce.cpp index 02fb56e2d..ed16ec6c5 100644 --- a/src/common/backend/parser/parse_coerce.cpp +++ b/src/common/backend/parser/parse_coerce.cpp @@ -212,20 +212,23 @@ Node* coerce_to_target_type(ParseState* pstate, Node* expr, Oid exprtype, Oid ta (cformat != COERCE_IMPLICIT_CAST), (result != expr && !IsA(result, Const))); } + + if (expr != origexpr && ( #ifdef PGXC - /* Do not need to do that on local Coordinator */ - if (IsConnFromCoord()) + /* Do not need to do that on local Coordinator */ + IsConnFromCoord() || #endif - if (expr != origexpr) { - /* Reinstall top CollateExpr */ - CollateExpr* coll = (CollateExpr*)origexpr; - CollateExpr* newcoll = makeNode(CollateExpr); + type_is_collatable(targettype))) { - newcoll->arg = (Expr*)result; - newcoll->collOid = coll->collOid; - newcoll->location = coll->location; - result = (Node*)newcoll; - } + /* Reinstall top CollateExpr */ + CollateExpr* coll = (CollateExpr*)origexpr; + CollateExpr* newcoll = makeNode(CollateExpr); + + newcoll->arg = (Expr*)result; + newcoll->collOid = coll->collOid; + newcoll->location = coll->location; + result = (Node*)newcoll; + } return result; } diff --git a/src/test/regress/expected/charset_utf8mb4_b_db.out b/src/test/regress/expected/charset_utf8mb4_b_db.out index 736fe0c22..5fead0c32 100644 --- a/src/test/regress/expected/charset_utf8mb4_b_db.out +++ b/src/test/regress/expected/charset_utf8mb4_b_db.out @@ -371,6 +371,63 @@ SELECT '高斯' COLLATE "binary"; -- ERROR ERROR: COLLATION "binary" is not valid for CHARACTER SET "UTF8" LINE 1: SELECT '高斯' COLLATE "binary"; ^ +-- test CollateExpr +CREATE TABLE t_collate_expr( + ftext text collate utf8mb4_bin, + fbytea bytea, + fvbit varbit(8), + fint int +); +-- -- test INSERT +INSERT INTO t_collate_expr(ftext) VALUES('01100001' collate "binary"); -- ERROR +ERROR: COLLATION "binary" is not valid for CHARACTER SET "UTF8" +LINE 1: ...SERT INTO t_collate_expr(ftext) VALUES('01100001' collate "b... + ^ +CONTEXT: referenced column: ftext +INSERT INTO t_collate_expr(ftext) VALUES('01100001' collate gbk_bin); -- ERROR +ERROR: COLLATION "gbk_bin" is not valid for CHARACTER SET "UTF8" +LINE 1: ...SERT INTO t_collate_expr(ftext) VALUES('01100001' collate gb... + ^ +CONTEXT: referenced column: ftext +INSERT INTO t_collate_expr(ftext) VALUES('01100001' collate utf8mb4_unicode_ci); +INSERT INTO t_collate_expr(ftext) VALUES('01100001' collate gbk_bin collate utf8mb4_unicode_ci); -- only reserve top collate +INSERT INTO t_collate_expr(fbytea) VALUES('01100001' collate "binary"); -- do not check collate +INSERT INTO t_collate_expr(fbytea) VALUES('01100001' collate gbk_bin); -- do not check collate +INSERT INTO t_collate_expr(fbytea) VALUES('01100001' collate utf8mb4_unicode_ci); +INSERT INTO t_collate_expr(fvbit) VALUES('01100001' collate "binary"); -- do not check collate +INSERT INTO t_collate_expr(fvbit) VALUES('01100001' collate gbk_bin); -- do not check collate +INSERT INTO t_collate_expr(fvbit) VALUES('01100001' collate utf8mb4_unicode_ci); +INSERT INTO t_collate_expr(fint) VALUES('01100001' collate "binary"); -- do not check collate +INSERT INTO t_collate_expr(fint) VALUES('01100001' collate gbk_bin); -- do not check collate +INSERT INTO t_collate_expr(fint) VALUES('01100001' collate utf8mb4_unicode_ci); +INSERT INTO t_collate_expr(fbytea) VALUES('01100001' collate gbk_bin collate utf8mb4_unicode_ci); -- do not check collate +INSERT INTO t_collate_expr(fbytea) VALUES('01100001' collate utf8mb4_general_ci collate gbk_bin); -- do not check collate +INSERT INTO t_collate_expr(fvbit) VALUES('01100001' collate gbk_bin collate utf8mb4_unicode_ci); -- do not check collate +INSERT INTO t_collate_expr(fvbit) VALUES('01100001' collate utf8mb4_general_ci collate gbk_bin); -- do not check collate +INSERT INTO t_collate_expr(fint) VALUES('01100001' collate gbk_bin collate utf8mb4_unicode_ci); -- do not check collate +INSERT INTO t_collate_expr(fint) VALUES('01100001' collate utf8mb4_general_ci collate gbk_bin); -- do not check collate +-- -- test limit +select 1 from t_collate_expr limit(to_hex('11') collate "binary"); + ?column? +---------- +(0 rows) + +select 1 from t_collate_expr limit(to_hex('11') collate gbk_bin); + ?column? +---------- +(0 rows) + +select 1 from t_collate_expr limit(to_hex('11') collate utf8mb4_unicode_ci); + ?column? +---------- +(0 rows) + +select 1 from t_collate_expr limit(to_hex('11') collate gbk_bin collate utf8mb4_unicode_ci); -- do not check collate + ?column? +---------- +(0 rows) + +DROP TABLE t_collate_expr; -- 中文 const charset SELECT CAST('高斯' AS bytea); bytea @@ -2287,18 +2344,16 @@ EXECUTE test_merge_collation(_utf8mb4'高斯DB'); -- $1 use collation_connection 高斯DB高斯DB | utf8mb4_general_ci (1 row) -EXECUTE test_merge_collation(_utf8mb4'高斯DB' collate gbk_chinese_ci); -- explicit noneffective, utf8_gen - result | pg_collation_for ---------------+-------------------- - 高斯DB高斯DB | utf8mb4_general_ci -(1 row) - EXECUTE test_merge_collation(_gbk'高斯DB'); -- _gbk noneffective, utf8_gen result | pg_collation_for --------------+-------------------- 高斯DB高斯DB | utf8mb4_general_ci (1 row) +EXECUTE test_merge_collation(_utf8mb4'高斯DB' collate gbk_chinese_ci); -- implicit type cast, keep explicit collation and check it, ERROR +ERROR: COLLATION "gbk_chinese_ci" is not valid for CHARACTER SET "UTF8" +LINE 1: EXECUTE test_merge_collation(_utf8mb4'高斯DB' collate gbk_ch... + ^ DEALLOCATE test_merge_collation; -- -- -- -- PBE with explicit collation, PREPARE test_merge_collation(text) AS diff --git a/src/test/regress/sql/charset_utf8mb4_b_db.sql b/src/test/regress/sql/charset_utf8mb4_b_db.sql index bf0dd844a..a6ab68d58 100644 --- a/src/test/regress/sql/charset_utf8mb4_b_db.sql +++ b/src/test/regress/sql/charset_utf8mb4_b_db.sql @@ -91,6 +91,42 @@ SELECT '高斯' COLLATE "gbk_chinese_ci"; -- ERROR SELECT '高斯' COLLATE "gb18030_chinese_ci"; -- ERROR SELECT '高斯' COLLATE "binary"; -- ERROR +-- test CollateExpr +CREATE TABLE t_collate_expr( + ftext text collate utf8mb4_bin, + fbytea bytea, + fvbit varbit(8), + fint int +); +-- -- test INSERT +INSERT INTO t_collate_expr(ftext) VALUES('01100001' collate "binary"); -- ERROR +INSERT INTO t_collate_expr(ftext) VALUES('01100001' collate gbk_bin); -- ERROR +INSERT INTO t_collate_expr(ftext) VALUES('01100001' collate utf8mb4_unicode_ci); +INSERT INTO t_collate_expr(ftext) VALUES('01100001' collate gbk_bin collate utf8mb4_unicode_ci); -- only reserve top collate +INSERT INTO t_collate_expr(fbytea) VALUES('01100001' collate "binary"); -- do not check collate +INSERT INTO t_collate_expr(fbytea) VALUES('01100001' collate gbk_bin); -- do not check collate +INSERT INTO t_collate_expr(fbytea) VALUES('01100001' collate utf8mb4_unicode_ci); +INSERT INTO t_collate_expr(fvbit) VALUES('01100001' collate "binary"); -- do not check collate +INSERT INTO t_collate_expr(fvbit) VALUES('01100001' collate gbk_bin); -- do not check collate +INSERT INTO t_collate_expr(fvbit) VALUES('01100001' collate utf8mb4_unicode_ci); +INSERT INTO t_collate_expr(fint) VALUES('01100001' collate "binary"); -- do not check collate +INSERT INTO t_collate_expr(fint) VALUES('01100001' collate gbk_bin); -- do not check collate +INSERT INTO t_collate_expr(fint) VALUES('01100001' collate utf8mb4_unicode_ci); +INSERT INTO t_collate_expr(fbytea) VALUES('01100001' collate gbk_bin collate utf8mb4_unicode_ci); -- do not check collate +INSERT INTO t_collate_expr(fbytea) VALUES('01100001' collate utf8mb4_general_ci collate gbk_bin); -- do not check collate +INSERT INTO t_collate_expr(fvbit) VALUES('01100001' collate gbk_bin collate utf8mb4_unicode_ci); -- do not check collate +INSERT INTO t_collate_expr(fvbit) VALUES('01100001' collate utf8mb4_general_ci collate gbk_bin); -- do not check collate +INSERT INTO t_collate_expr(fint) VALUES('01100001' collate gbk_bin collate utf8mb4_unicode_ci); -- do not check collate +INSERT INTO t_collate_expr(fint) VALUES('01100001' collate utf8mb4_general_ci collate gbk_bin); -- do not check collate + +-- -- test limit +select 1 from t_collate_expr limit(to_hex('11') collate "binary"); +select 1 from t_collate_expr limit(to_hex('11') collate gbk_bin); +select 1 from t_collate_expr limit(to_hex('11') collate utf8mb4_unicode_ci); +select 1 from t_collate_expr limit(to_hex('11') collate gbk_bin collate utf8mb4_unicode_ci); -- do not check collate + +DROP TABLE t_collate_expr; + -- 中文 const charset SELECT CAST('高斯' AS bytea); SELECT CAST(_binary'高斯' AS bytea); @@ -544,8 +580,8 @@ DEALLOCATE test_merge_collation; PREPARE test_merge_collation(text) AS SELECT CONCAT($1, fgbk_bin) result, collation for(result) FROM t_diff_charset_columns; EXECUTE test_merge_collation(_utf8mb4'高斯DB'); -- $1 use collation_connection, utf8_gen -EXECUTE test_merge_collation(_utf8mb4'高斯DB' collate gbk_chinese_ci); -- explicit noneffective, utf8_gen EXECUTE test_merge_collation(_gbk'高斯DB'); -- _gbk noneffective, utf8_gen +EXECUTE test_merge_collation(_utf8mb4'高斯DB' collate gbk_chinese_ci); -- implicit type cast, keep explicit collation and check it, ERROR DEALLOCATE test_merge_collation; -- -- -- -- PBE with explicit collation, PREPARE test_merge_collation(text) AS From b5befdb9f8e617ebfce5856e02fe5fc3203b38bb Mon Sep 17 00:00:00 2001 From: movead Date: Fri, 28 Jul 2023 20:56:15 +0800 Subject: [PATCH 100/304] enable group dispatch wal records for parallel redo and cancle wait during get snapshot on standby --- src/bin/gs_guc/cluster_guc.conf | 4 ++ .../backend/utils/misc/guc/guc_storage.cpp | 56 +++++++++++++++++++ .../access/transam/extreme_rto/dispatcher.cpp | 3 +- .../transam/parallel_recovery/dispatcher.cpp | 52 ++++++++++++++--- .../transam/parallel_recovery/page_redo.cpp | 46 +++++++++++++++ .../transam/parallel_recovery/txn_redo.cpp | 11 ++++ src/gausskernel/storage/ipc/procarray.cpp | 3 +- src/include/access/multi_redo_api.h | 11 +++- .../access/parallel_recovery/dispatcher.h | 4 +- .../access/parallel_recovery/txn_redo.h | 1 + .../knl/knl_guc/knl_instance_attr_storage.h | 4 ++ .../regress/output/recovery_2pc_tools.source | 4 ++ 12 files changed, 186 insertions(+), 13 deletions(-) diff --git a/src/bin/gs_guc/cluster_guc.conf b/src/bin/gs_guc/cluster_guc.conf index 986e134d3..76edca7b4 100755 --- a/src/bin/gs_guc/cluster_guc.conf +++ b/src/bin/gs_guc/cluster_guc.conf @@ -33,6 +33,8 @@ alarm_component|string|0,0|NULL|NULL| alarm_report_interval|int|0,2147483647|NULL|NULL| allow_concurrent_tuple_update|bool|0,0|NULL|NULL| enable_huge_pages|bool|0,0|NULL|NULL| +parallel_recovery_cost_record|bool|0,0|NULL|NULL| +enable_batch_dispatch|bool|0,0|NULL|NULL| allow_create_sysobject|bool|0,0|NULL|NULL| allow_system_table_mods|bool|0,0|NULL|NULL| application_name|string|0,0|NULL|NULL| @@ -665,6 +667,8 @@ pagewriter_thread_num|int|1,16|NULL|NULL| audit_thread_num|int|1,48|NULL|NULL| dw_file_num|int|1,16|NULL|NULL| dw_file_size|int|32,256|NULL|NULL| +parallel_recovery_batch|int|1,100000|NULL|NULL| +parallel_recovery_timeout|int|1,1000|ms|NULL| incremental_checkpoint_timeout|int|1,3600|s|NULL| enable_incremental_checkpoint|bool|0,0|NULL|NULL| enable_double_write|bool|0,0|NULL|NULL| diff --git a/src/common/backend/utils/misc/guc/guc_storage.cpp b/src/common/backend/utils/misc/guc/guc_storage.cpp index 7306eca39..d016106f2 100755 --- a/src/common/backend/utils/misc/guc/guc_storage.cpp +++ b/src/common/backend/utils/misc/guc/guc_storage.cpp @@ -1196,6 +1196,31 @@ static void InitStorageConfigureNamesBool() NULL, NULL, NULL}, + + {{"parallel_recovery_cost_record", + PGC_POSTMASTER, + NODE_SINGLENODE, + RESOURCES_RECOVERY, + gettext_noop("Record process time in every stage of parallel reovery"), + NULL}, + &g_instance.attr.attr_storage.parallel_recovery_cost_record, + false, + NULL, + NULL, + NULL}, + + {{"enable_batch_dispatch", + PGC_POSTMASTER, + NODE_SINGLENODE, + RESOURCES_RECOVERY, + gettext_noop("Enable batch dispatch for parallel reovery"), + NULL}, + &g_instance.attr.attr_storage.enable_batch_dispatch, + true, + NULL, + NULL, + NULL}, + /* End-of-list marker */ {{NULL, (GucContext)0, @@ -3092,6 +3117,22 @@ static void InitStorageConfigureNamesInt() NULL, NULL, NULL}, + + {{"parallel_recovery_batch", + PGC_SIGHUP, + NODE_SINGLENODE, + RESOURCES_RECOVERY, + gettext_noop("Set the batch that starup thread hold in parallel recovery."), + NULL, + 0}, + &g_instance.attr.attr_storage.parallel_recovery_batch, + 1000, + 1, + 100000, + NULL, + NULL, + NULL}, + {{"incremental_checkpoint_timeout", PGC_SIGHUP, NODE_ALL, @@ -3682,6 +3723,20 @@ static void InitStorageConfigureNamesInt() check_ss_txnstatus_cache_size, NULL, NULL}, + {{"parallel_recovery_timeout", + PGC_SIGHUP, + NODE_SINGLENODE, + RESOURCES_RECOVERY, + gettext_noop("parallel recovery timeout."), + NULL, + GUC_UNIT_MS}, + &g_instance.attr.attr_storage.parallel_recovery_timeout, + 300, + 1, + 1000, + NULL, + NULL, + NULL}, /* End-of-list marker */ {{NULL, (GucContext)0, @@ -3830,6 +3885,7 @@ static void InitStorageConfigureNamesReal() NULL, NULL, NULL}, + /* End-of-list marker */ {{NULL, (GucContext)0, diff --git a/src/gausskernel/storage/access/transam/extreme_rto/dispatcher.cpp b/src/gausskernel/storage/access/transam/extreme_rto/dispatcher.cpp index 42382170c..a881731e1 100755 --- a/src/gausskernel/storage/access/transam/extreme_rto/dispatcher.cpp +++ b/src/gausskernel/storage/access/transam/extreme_rto/dispatcher.cpp @@ -96,7 +96,6 @@ LogDispatcher *g_dispatcher = NULL; static const int XLOG_INFO_SHIFT_SIZE = 4; /* xlog info flag shift size */ static const int32 MAX_PENDING = 1; -static const int32 MAX_PENDING_STANDBY = 1; static const int32 ITEM_QUQUE_SIZE_RATIO = 5; static const uint32 EXIT_WAIT_DELAY = 100; /* 100 us */ @@ -2139,7 +2138,7 @@ void redo_get_worker_time_count(RedoWorkerTimeCountsInfo **workerCountInfoList, knl_parallel_redo_state state = g_instance.comm_cxt.predo_cxt.state; SpinLockRelease(&(g_instance.comm_cxt.predo_cxt.rwlock)); - if (state != REDO_IN_PROGRESS) { + if (state != REDO_IN_PROGRESS || !g_instance.attr.attr_storage.parallel_recovery_cost_record) { *realNum = 0; return; } diff --git a/src/gausskernel/storage/access/transam/parallel_recovery/dispatcher.cpp b/src/gausskernel/storage/access/transam/parallel_recovery/dispatcher.cpp index 5a23e140d..a77019908 100755 --- a/src/gausskernel/storage/access/transam/parallel_recovery/dispatcher.cpp +++ b/src/gausskernel/storage/access/transam/parallel_recovery/dispatcher.cpp @@ -96,7 +96,6 @@ LogDispatcher *g_dispatcher = NULL; static const int XLOG_INFO_SHIFT_SIZE = 4; /* xlog info flag shift size */ static const int32 MAX_PENDING = 1; -static const int32 MAX_PENDING_STANDBY = 1; static const int32 ITEM_QUQUE_SIZE_RATIO = 5; static const uint32 EXIT_WAIT_DELAY = 100; /* 100 us */ @@ -167,6 +166,7 @@ static bool DispatchUndoActionRecord(XLogReaderState *record, List *expectedTLIs static bool DispatchRollbackFinishRecord(XLogReaderState *record, List *expectedTLIs, TimestampTz recordXTime); static uint32 GetUndoSpaceWorkerId(int zid); static void HandleStartupProcInterruptsForParallelRedo(void); +static bool timeoutForDispatch(void); RedoWaitInfo redo_get_io_event(int32 event_id); @@ -384,16 +384,14 @@ static LogDispatcher *CreateDispatcher() SpinLockAcquire(&(g_instance.comm_cxt.predo_cxt.rwlock)); g_instance.comm_cxt.predo_cxt.state = REDO_STARTING_BEGIN; SpinLockRelease(&(g_instance.comm_cxt.predo_cxt.rwlock)); - if (OnHotStandBy()) - newDispatcher->pendingMax = MAX_PENDING_STANDBY; - else - newDispatcher->pendingMax = MAX_PENDING; /* one batch, one recorder */ + newDispatcher->totalCostTime = 0; newDispatcher->txnCostTime = 0; newDispatcher->pprCostTime = 0; newDispatcher->dispatchReadRecPtr = 0; newDispatcher->dispatchEndRecPtr = 0; newDispatcher->startupTimeCost = t_thrd.xlog_cxt.timeCost; + newDispatcher->full_sync_dispatch = !g_instance.attr.attr_storage.enable_batch_dispatch; return newDispatcher; } @@ -560,6 +558,22 @@ static bool RmgrGistRecordInfoValid(XLogReaderState *record, uint8 minInfo, uint return false; } +static bool timeoutForDispatch(void) +{ + int parallel_recovery_timeout = 0; + TimestampTz current_time = 0; + TimestampTz dispatch_limit_time = 0; + + current_time = GetCurrentTimestamp(); + + parallel_recovery_timeout = g_instance.attr.attr_storage.parallel_recovery_timeout; + dispatch_limit_time = TimestampTzPlusMilliseconds(g_dispatcher->lastDispatchTime, + parallel_recovery_timeout); + if(current_time >= dispatch_limit_time) + return true; + return false; +} + void CheckDispatchCount(XLogRecPtr lastCheckLsn) { uint64 maxCount = 0; @@ -592,6 +606,8 @@ void CheckDispatchCount(XLogRecPtr lastCheckLsn) } } + + /* Run from the dispatcher thread. */ void DispatchRedoRecordToFile(XLogReaderState *record, List *expectedTLIs, TimestampTz recordXTime) { @@ -600,6 +616,8 @@ void DispatchRedoRecordToFile(XLogReaderState *record, List *expectedTLIs, Times uint32 indexid = (uint32)-1; uint32 rmid = XLogRecGetRmid(record); uint32 term = XLogRecGetTerm(record); + int dispatch_batch = 0; + if (term > g_instance.comm_cxt.localinfo_cxt.term_from_xlog) { g_instance.comm_cxt.localinfo_cxt.term_from_xlog = term; } @@ -629,9 +647,11 @@ void DispatchRedoRecordToFile(XLogReaderState *record, List *expectedTLIs, Times g_dispatcher->dispatchReadRecPtr = record->ReadRecPtr; g_dispatcher->dispatchEndRecPtr = record->EndRecPtr; + dispatch_batch = g_instance.attr.attr_storage.enable_batch_dispatch ? + g_instance.attr.attr_storage.parallel_recovery_batch : 1; if (isNeedFullSync) ProcessPendingRecords(true); - else if (++g_dispatcher->pendingCount >= g_dispatcher->pendingMax) + else if (++g_dispatcher->pendingCount >= dispatch_batch || timeoutForDispatch()) ProcessPendingRecords(); if (fatalerror == true) { @@ -1491,6 +1511,9 @@ static bool StandbyWillChangeStandbyState(XLogReaderState *record) /* : false not wait for other workers */ void ProcessPendingRecords(bool fullSync) { + if(fullSync) + g_dispatcher->full_sync_dispatch = true; + g_dispatcher->lastDispatchTime = GetCurrentTimestamp(); if ((get_real_recovery_parallelism() > 1) && (GetPageWorkerCount() > 0)) { for (uint32 i = 0; i < g_dispatcher->pageWorkerCount; i++) { uint64 blockcnt = 0; @@ -1516,6 +1539,8 @@ void ProcessPendingRecords(bool fullSync) ApplyReadyTxnLogRecords(g_dispatcher->txnWorker, fullSync); g_dispatcher->pendingCount = 0; } + if(fullSync) + g_dispatcher->full_sync_dispatch = false; } /* Run from the dispatcher thread. */ @@ -1524,7 +1549,10 @@ void ProcessPendingRecords(bool fullSync) void ProcessTrxnRecords(bool fullSync) { if ((get_real_recovery_parallelism() > 1) && (GetPageWorkerCount() > 0)) { - ApplyReadyTxnLogRecords(g_dispatcher->txnWorker, fullSync); + if (g_instance.attr.attr_storage.enable_batch_dispatch) + ProcessPendingRecords(fullSync); + else + ApplyReadyTxnLogRecords(g_dispatcher->txnWorker, fullSync); if (fullSync && (IsTxnWorkerIdle(g_dispatcher->txnWorker))) { /* notify pageworker sleep long time */ @@ -1980,7 +2008,7 @@ void redo_get_worker_time_count(RedoWorkerTimeCountsInfo **workerCountInfoList, knl_parallel_redo_state state = g_instance.comm_cxt.predo_cxt.state; SpinLockRelease(&(g_instance.comm_cxt.predo_cxt.rwlock)); - if (state != REDO_IN_PROGRESS) { + if (state != REDO_IN_PROGRESS || !g_instance.attr.attr_storage.parallel_recovery_cost_record) { *realNum = 0; return; } @@ -2501,4 +2529,12 @@ static void HandleStartupProcInterruptsForParallelRedo(void) if (IsUnderPostmaster && !PostmasterIsAlive()) gs_thread_exit(1); } + +bool in_full_sync_dispatch(void) +{ + if (!g_dispatcher || !g_instance.attr.attr_storage.enable_batch_dispatch) + return true; + return g_dispatcher->full_sync_dispatch; +} + } diff --git a/src/gausskernel/storage/access/transam/parallel_recovery/page_redo.cpp b/src/gausskernel/storage/access/transam/parallel_recovery/page_redo.cpp index c9b465f22..b49b081e8 100755 --- a/src/gausskernel/storage/access/transam/parallel_recovery/page_redo.cpp +++ b/src/gausskernel/storage/access/transam/parallel_recovery/page_redo.cpp @@ -555,12 +555,58 @@ static void ApplyRecordWithoutSyncUndoLog(RedoItem *item) } } +/* + * If woker do a page vacuum redo, it should wait if it's operator + * may cause snapshot invalid. + */ +static void wait_valid_snapshot(XLogReaderState *record) +{ + RmgrId rm_id = XLogRecGetRmid(record); + uint8 info = (XLogRecGetInfo(record) & ~XLR_INFO_MASK) & XLOG_HEAP_OPMASK; + xl_heap_clean* xlrec = NULL; + uint64 blockcnt = 0; + + if(rm_id != RM_HEAP2_ID || info != XLOG_HEAP2_CLEAN) + return; + + xlrec = (xl_heap_clean*)XLogRecGetData(record); + + /* + * If xlrec->latestRemovedXid <= t_thrd.xact_cxt.ShmemVariableCache->standbyXmin then + * it will not incluence current snapshot, so it can exec redo. + */ + while(t_thrd.xact_cxt.ShmemVariableCache->standbyXmin < xlrec->latestRemovedXid && + !in_full_sync_dispatch()) { + /* + * Normaly, it need wait for startup thread handle xact wal records, but there be a case + * that if a very old xid commit and no new xact comes then xlrec->latestRemovedXid > + * t_thrd.xact_cxt.ShmemVariableCache->standbyXmin all the time. + * + * So if we do not have new xact work in startup thread, it avoid wait. + */ + if (getTransedTxnLsn(g_dispatcher->txnWorker) < record->EndRecPtr) + return; + pg_usleep(10); + blockcnt++; + if ((blockcnt & OUTPUT_WAIT_COUNT) == OUTPUT_WAIT_COUNT) { + XLogRecPtr LatestReplayedRecPtr = GetXLogReplayRecPtr(NULL); + ereport(WARNING, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), + errmsg("[REDO_LOG_TRACE]wait_valid_snapshot:recordEndLsn:%lu, blockcnt:%lu, " + "Workerid:%u, LatestReplayedRecPtr:%lu", + record->EndRecPtr, blockcnt, g_redoWorker->id, LatestReplayedRecPtr))); + } + RedoInterruptCallBack(); + } +} + /* Run from the worker thread. */ static void ApplySinglePageRecord(RedoItem *item, bool replayUndo) { XLogReaderState *record = &item->record; long readbufcountbefore = u_sess->instr_cxt.pg_buffer_usage->local_blks_read; MemoryContext oldCtx = MemoryContextSwitchTo(g_redoWorker->oldCtx); + + wait_valid_snapshot(record); ApplyRedoRecord(record); (void)MemoryContextSwitchTo(oldCtx); record->readblocks = u_sess->instr_cxt.pg_buffer_usage->local_blks_read - readbufcountbefore; diff --git a/src/gausskernel/storage/access/transam/parallel_recovery/txn_redo.cpp b/src/gausskernel/storage/access/transam/parallel_recovery/txn_redo.cpp index 0c707c40f..141671aa1 100644 --- a/src/gausskernel/storage/access/transam/parallel_recovery/txn_redo.cpp +++ b/src/gausskernel/storage/access/transam/parallel_recovery/txn_redo.cpp @@ -61,8 +61,15 @@ struct TxnRedoWorker { RedoItem *pendingTail; /* The tail of the RedoItem list. */ RedoItem *procHead; RedoItem *procTail; + XLogRecPtr dispatched_txn_lsn; + XLogRecPtr transed_txn_lsn; }; +XLogRecPtr getTransedTxnLsn(TxnRedoWorker *worker) +{ + return (XLogRecPtr)pg_atomic_read_u64((volatile uint64*)&worker->transed_txn_lsn); +} + TxnRedoWorker *StartTxnRedoWorker() { TxnRedoWorker *worker = (TxnRedoWorker *)palloc(sizeof(TxnRedoWorker)); @@ -71,6 +78,8 @@ TxnRedoWorker *StartTxnRedoWorker() worker->procHead = NULL; worker->procTail = NULL; + worker->dispatched_txn_lsn = 0; + worker->transed_txn_lsn = 0; return worker; } @@ -90,6 +99,7 @@ void AddTxnRedoItem(TxnRedoWorker *worker, RedoItem *item) * TxnRedoItems are never shared with other workers. * Simply use the next pointer for worker 0. */ + worker->dispatched_txn_lsn = item->record.EndRecPtr; if (worker->pendingHead == NULL) { worker->pendingHead = item; } else { @@ -210,6 +220,7 @@ void MoveTxnItemToApplyQueue(TxnRedoWorker *worker) worker->procTail = worker->pendingTail; worker->pendingHead = NULL; worker->pendingTail = NULL; + pg_atomic_write_u64(&worker->transed_txn_lsn, worker->dispatched_txn_lsn); } static RedoItem *ProcTxnItem(RedoItem *item) diff --git a/src/gausskernel/storage/ipc/procarray.cpp b/src/gausskernel/storage/ipc/procarray.cpp index ab7216d0a..db23b33fe 100755 --- a/src/gausskernel/storage/ipc/procarray.cpp +++ b/src/gausskernel/storage/ipc/procarray.cpp @@ -2160,7 +2160,8 @@ Snapshot GetSnapshotData(Snapshot snapshot, bool force_local_snapshot) } else if (LWLockConditionalAcquire(ProcArrayLock, LW_EXCLUSIVE)) { if ((t_thrd.xact_cxt.ShmemVariableCache->standbyXmin <= t_thrd.xact_cxt.ShmemVariableCache->standbyRedoCleanupXmin) && - (t_thrd.xact_cxt.ShmemVariableCache->standbyRedoCleanupXminLsn > redoEndLsn)) { + (t_thrd.xact_cxt.ShmemVariableCache->standbyRedoCleanupXminLsn > redoEndLsn) && + parallel_recovery::in_full_sync_dispatch()) { LWLockRelease(ProcArrayLock); retry_get = true; goto RETRY_GET; diff --git a/src/include/access/multi_redo_api.h b/src/include/access/multi_redo_api.h index 1471c37e6..eccb4e6b0 100644 --- a/src/include/access/multi_redo_api.h +++ b/src/include/access/multi_redo_api.h @@ -127,18 +127,27 @@ void ResetXLogStatics(); static inline void GetRedoStartTime(RedoTimeCost &cost) { + if(!g_instance.attr.attr_storage.parallel_recovery_cost_record) + return; cost.startTime = GetCurrentTimestamp(); } static inline void CountRedoTime(RedoTimeCost &cost) { + if(!g_instance.attr.attr_storage.parallel_recovery_cost_record) + return; cost.totalDuration += GetCurrentTimestamp() - cost.startTime; cost.counter += 1; } static inline void CountAndGetRedoTime(RedoTimeCost &curCost, RedoTimeCost &nextCost) { - uint64 curTime = GetCurrentTimestamp(); + uint64 curTime = 0; + + if(!g_instance.attr.attr_storage.parallel_recovery_cost_record) + return; + + curTime = GetCurrentTimestamp(); curCost.totalDuration += curTime - curCost.startTime; curCost.counter += 1; nextCost.startTime = curTime; diff --git a/src/include/access/parallel_recovery/dispatcher.h b/src/include/access/parallel_recovery/dispatcher.h index cad5d9481..047c36002 100644 --- a/src/include/access/parallel_recovery/dispatcher.h +++ b/src/include/access/parallel_recovery/dispatcher.h @@ -64,6 +64,7 @@ typedef struct LogDispatcher { uint64 pprCostTime; uint32 maxItemNum; uint32 curItemNum; + TimestampTz lastDispatchTime; /* last time we dispatch record list to works */ uint32* chosedWorkerIds; uint32 chosedWorkerCount; @@ -75,6 +76,7 @@ typedef struct LogDispatcher { XLogRedoNumStatics xlogStatics[RM_NEXT_ID][MAX_XLOG_INFO_NUM]; RedoTimeCost *startupTimeCost; DispatchFix dispatchFix; + bool full_sync_dispatch; } LogDispatcher; extern LogDispatcher* g_dispatcher; @@ -140,7 +142,7 @@ extern void CopyDataFromOldReader(XLogReaderState *newReaderState, XLogReaderSta bool TxnQueueIsEmpty(TxnRedoWorker* worker); void redo_get_worker_time_count(RedoWorkerTimeCountsInfo **workerCountInfoList, uint32 *realNum); - +bool in_full_sync_dispatch(void); } #endif diff --git a/src/include/access/parallel_recovery/txn_redo.h b/src/include/access/parallel_recovery/txn_redo.h index bfbd04b1b..6c1cb3fe4 100644 --- a/src/include/access/parallel_recovery/txn_redo.h +++ b/src/include/access/parallel_recovery/txn_redo.h @@ -38,5 +38,6 @@ void ApplyReadyTxnLogRecords(TxnRedoWorker* worker, bool forceAll); void MoveTxnItemToApplyQueue(TxnRedoWorker* worker); void DumpTxnWorker(TxnRedoWorker* txnWorker); bool IsTxnWorkerIdle(TxnRedoWorker* worker); +XLogRecPtr getTransedTxnLsn(TxnRedoWorker *worker); } #endif diff --git a/src/include/knl/knl_guc/knl_instance_attr_storage.h b/src/include/knl/knl_guc/knl_instance_attr_storage.h index 8d17c7e67..059ccb258 100755 --- a/src/include/knl/knl_guc/knl_instance_attr_storage.h +++ b/src/include/knl/knl_guc/knl_instance_attr_storage.h @@ -212,6 +212,10 @@ typedef struct knl_instance_attr_storage { #endif bool enable_huge_pages; int huge_page_size; + bool parallel_recovery_cost_record; + bool enable_batch_dispatch; + int parallel_recovery_timeout; + int parallel_recovery_batch; } knl_instance_attr_storage; #endif /* SRC_INCLUDE_KNL_KNL_INSTANCE_ATTR_STORAGE_H_ */ diff --git a/src/test/regress/output/recovery_2pc_tools.source b/src/test/regress/output/recovery_2pc_tools.source index 76c79865f..70b55f083 100644 --- a/src/test/regress/output/recovery_2pc_tools.source +++ b/src/test/regress/output/recovery_2pc_tools.source @@ -226,6 +226,7 @@ select name,vartype,unit,min_val,max_val from pg_settings where name <> 'qunit_c enable_auto_clean_unique_sql | bool | | | enable_auto_explain | bool | | | enable_availablezone | bool | | | + enable_batch_dispatch | bool | | | enable_bbox_dump | bool | | | enable_beta_features | bool | | | enable_beta_opfusion | bool | | | @@ -513,6 +514,9 @@ select name,vartype,unit,min_val,max_val from pg_settings where name <> 'qunit_c opfusion_debug_mode | enum | | | pagewriter_sleep | integer | ms | 0 | 3600000 pagewriter_thread_num | integer | | 1 | 16 + parallel_recovery_batch | integer | | 1 | 100000 + parallel_recovery_cost_record | bool | | | + parallel_recovery_timeout | integer | ms | 1 | 1000 partition_iterator_elimination | bool | | | partition_lock_upgrade_timeout | integer | | -1 | 3000 partition_max_cache_size | integer | kB | 4096 | 1073741823 From 5635f63c5f66733eacc40d01c1a9ddbc85424fb1 Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Mon, 7 Aug 2023 09:39:54 +0800 Subject: [PATCH 101/304] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E8=B5=84=E6=BA=90?= =?UTF-8?q?=E6=B1=A0=E5=8C=96=E5=8F=8C=E9=9B=86=E7=BE=A4switchover?= =?UTF-8?q?=E5=A4=9A=E6=AC=A1=E5=88=87=E6=8D=A2=E5=A4=B1=E8=B4=A5=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../process/postmaster/startup.cpp | 8 +++++++ .../storage/access/transam/xlog.cpp | 22 ++++--------------- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/src/gausskernel/process/postmaster/startup.cpp b/src/gausskernel/process/postmaster/startup.cpp index 25eb6b8b8..86a234741 100755 --- a/src/gausskernel/process/postmaster/startup.cpp +++ b/src/gausskernel/process/postmaster/startup.cpp @@ -231,6 +231,14 @@ void HandleStartupProcInterrupts(void) * Check if we were requested to exit without finishing recovery. */ if (t_thrd.startup_cxt.shutdown_requested && SmartShutdown != g_instance.status) { + if (t_thrd.xlog_cxt.StandbyModeRequested && IS_SHARED_STORAGE_STANDBY_CLUSTER && ENABLE_DMS) { + ereport(LOG, (errmsg("dorado standby cluster switchover shutdown startup\n"))); + if (!IsExtremeRedo()) { + DisownLatch(&t_thrd.shemem_ptr_cxt.XLogCtl->recoveryWakeupLatch); + } + DisownLatch(&t_thrd.shemem_ptr_cxt.XLogCtl->dataRecoveryLatch); + } + /* release compression ctx */ crps_destory_ctxs(); diff --git a/src/gausskernel/storage/access/transam/xlog.cpp b/src/gausskernel/storage/access/transam/xlog.cpp index 86389e038..9890b7e5d 100755 --- a/src/gausskernel/storage/access/transam/xlog.cpp +++ b/src/gausskernel/storage/access/transam/xlog.cpp @@ -19300,8 +19300,7 @@ static int SSReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int xlogctl->IsRecoveryDone = true; SpinLockRelease(&xlogctl->info_lck); static uint64 printFrequency = 0; - if (!(IS_SHARED_STORAGE_MODE) || - pg_atomic_read_u32(&t_thrd.walreceiverfuncs_cxt.WalRcv->rcvDoneFromShareStorage)) { + if (pg_atomic_read_u32(&t_thrd.walreceiverfuncs_cxt.WalRcv->rcvDoneFromShareStorage)) { knl_g_set_redo_finish_status(REDO_FINISH_STATUS_LOCAL | REDO_FINISH_STATUS_CM); if ((printFrequency & 0xFF) == 0) { ereport(LOG, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), @@ -19317,21 +19316,8 @@ static int SSReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int * If it hasn't been long since last attempt, sleep 1s to * avoid busy-waiting. */ - if (IS_SHARED_STORAGE_MODE) { - uint32 connMode = - pg_atomic_read_u32(&g_instance.comm_cxt.localinfo_cxt.need_disable_connection_node); - if (connMode) { - pg_atomic_write_u32(&g_instance.comm_cxt.localinfo_cxt.need_disable_connection_node, - false); - } - pg_usleep(2000000L); - } else { -#ifdef ENABLE_LITE_MODE - pg_usleep(1000000L); -#else - pg_usleep(50000L); -#endif - } + pg_atomic_write_u32(&g_instance.comm_cxt.localinfo_cxt.need_disable_connection_node, false); + pg_usleep(2000000L); } /* * If primary_conninfo is set, launch walreceiver to @@ -19347,7 +19333,7 @@ static int SSReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int */ load_server_mode(); - if (IS_SHARED_STORAGE_STANBY_MODE && !IS_SHARED_STORAGE_MAIN_STANDBY_MODE) { + if (DORADO_STANDBY_CLUSTER_MAINSTANDBY_NODE) { ProcTxnWorkLoad(false); /* use volatile pointer to prevent code rearrangement */ volatile WalRcvData *walrcv = t_thrd.walreceiverfuncs_cxt.WalRcv; From f1d89cc7ccb29029be3aaff179f373a50f7d13c2 Mon Sep 17 00:00:00 2001 From: hwhbj Date: Tue, 8 Aug 2023 09:55:52 +0800 Subject: [PATCH 102/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8DopenEuler=2022.03=20S?= =?UTF-8?q?P=E7=89=88=E6=9C=AC=E7=BC=96=E8=AF=91=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- configure | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure b/configure index 156025011..629c105b7 100755 --- a/configure +++ b/configure @@ -3279,7 +3279,7 @@ _ACEOF fi -if [ "$(cat /etc/system-release)" == "openEuler release 22.03 LTS" ]; then +if [ "$(cat /etc/system-release)" =~ ^"openEuler release 22.03".* ]; then with_openeuler_major=yes fi From 29de645aae0d0307e210bd84b835b26d48455845 Mon Sep 17 00:00:00 2001 From: cc_db_dev Date: Tue, 8 Aug 2023 10:16:40 +0800 Subject: [PATCH 103/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8DGetSynonymAndSchemaNa?= =?UTF-8?q?me=E4=B8=AD=E5=86=85=E5=AD=98use-after-free=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 描述： 应该对synForm->synname进行拷贝返回给调用者 --- src/common/backend/catalog/pg_synonym.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/backend/catalog/pg_synonym.cpp b/src/common/backend/catalog/pg_synonym.cpp index 77c31bc79..cd64a64cf 100644 --- a/src/common/backend/catalog/pg_synonym.cpp +++ b/src/common/backend/catalog/pg_synonym.cpp @@ -585,7 +585,7 @@ void GetSynonymAndSchemaName(Oid synOid, char** synName_p, char** synSchema_p) HeapTuple synTuple = SearchSysCache1(SYNOID, ObjectIdGetDatum(synOid)); if (HeapTupleIsValid(synTuple)) { Form_pg_synonym synForm = (Form_pg_synonym)GETSTRUCT(synTuple); - *synName_p = NameStr(synForm->synname); + *synName_p = pstrdup(NameStr(synForm->synname)); *synSchema_p = get_namespace_name(synForm->synnamespace); ReleaseSysCache(synTuple); From 3abc524ceebfd97d2a3683de45c3b8e23e6d8f9b Mon Sep 17 00:00:00 2001 From: kenxx Date: Mon, 7 Aug 2023 23:31:09 -0400 Subject: [PATCH 104/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/optimizer/sqladvisor/sqladvisor_online.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gausskernel/optimizer/sqladvisor/sqladvisor_online.cpp b/src/gausskernel/optimizer/sqladvisor/sqladvisor_online.cpp index 5561563a4..80a2c764d 100644 --- a/src/gausskernel/optimizer/sqladvisor/sqladvisor_online.cpp +++ b/src/gausskernel/optimizer/sqladvisor/sqladvisor_online.cpp @@ -320,7 +320,7 @@ static bool equalPLpgsqlFunc(PLpgSQL_function* funcA, PLpgSQL_function* funcB) /* equalPLpgEstate(funcA->cur_estate, funcB->cur_estate) */ for (int i = 0; i < funcA->fn_nargs; i++) { if (funcA->fn_argvarnos[i] != funcB->fn_argvarnos[i]) { - break; + return false; } } return true; From 255572e938db0570a4c667f60612001a0fead3c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=98ljy=E2=80=99?= Date: Tue, 8 Aug 2023 11:52:21 +0800 Subject: [PATCH 105/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E4=B8=80=E5=A4=84?= =?UTF-8?q?=E6=8F=92=E4=BB=B6=E5=B9=B6=E5=8F=91=E5=8A=A0=E8=BD=BD=E6=8F=92?= =?UTF-8?q?=E4=BB=B6=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/utils/init/postinit.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/common/backend/utils/init/postinit.cpp b/src/common/backend/utils/init/postinit.cpp index 4c8782aa4..8a8ea27b9 100644 --- a/src/common/backend/utils/init/postinit.cpp +++ b/src/common/backend/utils/init/postinit.cpp @@ -2828,6 +2828,9 @@ void PostgresInitializer::InitExtensionVariable() (void**)MemoryContextAllocZero(u_sess->self_mem_cxt, (Size)(initExtArraySize * sizeof(void*))); DynamicFileList* file_scanner = NULL; + AutoMutexLock libraryLock(&file_list_lock); + libraryLock.lock(); + for (file_scanner = file_list; file_scanner != NULL; file_scanner = file_scanner->next) { /* * If the library has a init_session_vars() function, call it for @@ -2835,8 +2838,10 @@ void PostgresInitializer::InitExtensionVariable() */ init_session_vars = (void(*)(void))pg_dlsym(file_scanner->handle, "init_session_vars"); if (init_session_vars != NULL) - (*init_session_vars)(); + (*init_session_vars)(); /*It is assumed that this does not cause a deadlock for the file_list_lock*/ } + + libraryLock.unLock(); /* check whether the extension has been created * at most one will be true. From 0355a1dd68d79ab8d3c79817bbeb384eaed3f6b5 Mon Sep 17 00:00:00 2001 From: xuxinxin Date: Tue, 8 Aug 2023 14:20:38 +0800 Subject: [PATCH 106/304] =?UTF-8?q?=E6=9B=B4=E6=96=B0dsscommit=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/ddes/ddes_commit_id | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gausskernel/ddes/ddes_commit_id b/src/gausskernel/ddes/ddes_commit_id index e1fdc9cb9..98b81de24 100644 --- a/src/gausskernel/ddes/ddes_commit_id +++ b/src/gausskernel/ddes/ddes_commit_id @@ -1,2 +1,2 @@ dms_commit_id=a9d399af041ae5491149fdb7c58e5115da3065b2 -dss_commit_id=8a4a7a5fb2bbe63b7596709e474459ff24359c08 +dss_commit_id=f9bb2fa60f02331b0373559776a8ebdba86c002f From e679e16cf04d361f700163e07e6e60e9e2f75a50 Mon Sep 17 00:00:00 2001 From: duzhuolin Date: Tue, 8 Aug 2023 14:40:45 +0800 Subject: [PATCH 107/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8Darchiver=E5=86=85?= =?UTF-8?q?=E5=AD=98=E6=B3=84=E6=BC=8F=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/process/postmaster/pgarch.cpp | 8 ++++++++ src/gausskernel/storage/replication/walsender.cpp | 4 +++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/gausskernel/process/postmaster/pgarch.cpp b/src/gausskernel/process/postmaster/pgarch.cpp index 64d2b6203..0b1b034f0 100755 --- a/src/gausskernel/process/postmaster/pgarch.cpp +++ b/src/gausskernel/process/postmaster/pgarch.cpp @@ -1012,6 +1012,14 @@ static void pgarch_archiveDone(const char* xlog) static void archKill(int code, Datum arg) { setObsArchLatch(NULL); + volatile WalSnd *walsnd = NULL; + for (int i = 0; i< g_instance.attr.attr_storage.max_wal_senders; i++) { + /* use volitile pointer to prevent code rearrangement */ + walsnd = &t_thrd.walsender_cxt.WalSndCtl->walsnds[i]; + SpinLockAcquire(&walsnd->mutex_archive_task_list); + walsnd->archive_task_count=0; + SpinLockRelease(&walsnd->mutex_archive_task_list); + } ereport(LOG, (errmsg("arch thread shut down, slotName: %s", t_thrd.arch.slot_name))); pfree_ext(t_thrd.arch.slot_name); if (t_thrd.arch.archive_config != NULL && t_thrd.arch.archive_config->archive_config.conn_config != NULL) { diff --git a/src/gausskernel/storage/replication/walsender.cpp b/src/gausskernel/storage/replication/walsender.cpp index 0da1ea0d9..af6f61347 100755 --- a/src/gausskernel/storage/replication/walsender.cpp +++ b/src/gausskernel/storage/replication/walsender.cpp @@ -7149,13 +7149,15 @@ void add_archive_task_to_list(int archive_task_status_idx, WalSnd *walsnd) ArchiveXlogMessage* get_archive_task_from_list() { volatile WalSnd *walsnd = t_thrd.walsender_cxt.MyWalSnd; + SpinLockAcquire(&walsnd->mutex_archive_task_list); volatile unsigned int *archive_task_count = &walsnd->archive_task_count; if (*archive_task_count == 0) { + SpinLockRelease(&walsnd->mutex_archive_task_list); return NULL; } ArchiveXlogMessage *result = NULL; int idx = -1; - SpinLockAcquire(&walsnd->mutex_archive_task_list); + idx = lfirst_int(list_head(walsnd->archive_task_list)); result = &g_instance.archive_obs_cxt.archive_status[idx].archive_task; walsnd->archive_task_list = list_delete_first(walsnd->archive_task_list); From d8e7ae442aa749ed2179e8b724102bb367c7324d Mon Sep 17 00:00:00 2001 From: duzhuolin Date: Tue, 8 Aug 2023 14:49:04 +0800 Subject: [PATCH 108/304] =?UTF-8?q?pageHack=E5=86=85=E5=AD=98=E6=B3=84?= =?UTF-8?q?=E6=BC=8F=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- contrib/pagehack/pagehack.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/contrib/pagehack/pagehack.cpp b/contrib/pagehack/pagehack.cpp index eab5c5186..8adaed889 100644 --- a/contrib/pagehack/pagehack.cpp +++ b/contrib/pagehack/pagehack.cpp @@ -3230,6 +3230,7 @@ static int parse_uncompressed_page_file(const char *filename, SegmentType type, /* parse */ number = CalculateMaxBlockNumber(blknum, start, number); if (number == InvalidBlockNumber) { + fclose(fd); return false; } else if ((start + number) > blknum) { fprintf(stderr, @@ -4492,6 +4493,8 @@ static bool parse_dw_file(const char* file_name, uint32 start_page, uint32 page_ if (start_page >= dw_batch_page_num) { fprintf(stdout, "start_page %u exceeds the double write file upper limit offset %u\n", start_page, dw_batch_page_num - 1); + free(dw_buf); + fclose(fd); return false; } file_head.start = (uint16)start_page; From 084dc9514423aba2afaf5a90c5f1d822e9d520cb Mon Sep 17 00:00:00 2001 From: duzhuolin Date: Tue, 8 Aug 2023 14:55:12 +0800 Subject: [PATCH 109/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8DrelationGetPartitionO?= =?UTF-8?q?idList=E5=87=BD=E6=95=B0=E8=BF=94=E5=9B=9E=E5=80=BC=E6=9C=AA?= =?UTF-8?q?=E5=88=A4=E7=A9=BA=E5=AD=98=E5=9C=A8=E5=BC=95=E7=94=A8=E7=A9=BA?= =?UTF-8?q?=E6=8C=87=E9=92=88=E9=A3=8E=E9=99=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/catalog/index.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/common/backend/catalog/index.cpp b/src/common/backend/catalog/index.cpp index 7e99bbf96..3cf78ce9c 100644 --- a/src/common/backend/catalog/index.cpp +++ b/src/common/backend/catalog/index.cpp @@ -4422,7 +4422,10 @@ double* GetGlobalIndexTuplesForSubPartition(Relation heapRelation, Relation inde partition = partitionOpen(heapRelation, partitionId, ShareLock); heapPartRel = partitionGetRelation(heapRelation, partition); subPartitionIdList = relationGetPartitionOidList(heapPartRel); - int subPartNum = subPartitionIdList->length; + int subPartNum = 0; + if (subPartitionIdList != NULL) { + subPartNum = subPartitionIdList->length; + } if (globalIndexTuples != NULL) { globalIndexTuples = (double*)repalloc(globalIndexTuples, (subPartitionIdx + subPartNum) * sizeof(double)); } else { From 80bd983fb922ea027656a33dbe2a3c1c330a15ea Mon Sep 17 00:00:00 2001 From: duzhuolin Date: Tue, 8 Aug 2023 14:59:59 +0800 Subject: [PATCH 110/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E8=A1=A8=E7=BA=A7?= =?UTF-8?q?=E7=BA=A6=E6=9D=9Fconkey=E4=B8=BA=E7=A9=BA=E6=97=B6=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E5=BC=82=E5=B8=B8=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/optimizer/plan/planner.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/gausskernel/optimizer/plan/planner.cpp b/src/gausskernel/optimizer/plan/planner.cpp index a0d4f193f..ad34b3abc 100755 --- a/src/gausskernel/optimizer/plan/planner.cpp +++ b/src/gausskernel/optimizer/plan/planner.cpp @@ -12119,6 +12119,9 @@ bool findConstraintByVar(Var* var, Oid relid, constraintType conType) ArrayType* arr = NULL; adatum = SysCacheGetAttr(CONSTROID, htup, Anum_pg_constraint_conkey, &isNull); + if (adatum == 0) { + continue; + } arr = DatumGetArrayTypeP(adatum); attnums = (int16*)ARR_DATA_PTR(arr); From 410cd003621fc47f6f1edc1013a6ab74ed6dd186 Mon Sep 17 00:00:00 2001 From: duzhuolin Date: Tue, 8 Aug 2023 15:13:13 +0800 Subject: [PATCH 111/304] =?UTF-8?q?bugfix=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/storage/access/transam/xlog.cpp | 4 ++-- src/gausskernel/storage/lmgr/lock.cpp | 2 +- src/gausskernel/storage/tcap/tcap_manager.cpp | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/gausskernel/storage/access/transam/xlog.cpp b/src/gausskernel/storage/access/transam/xlog.cpp index 015d48803..60373a889 100755 --- a/src/gausskernel/storage/access/transam/xlog.cpp +++ b/src/gausskernel/storage/access/transam/xlog.cpp @@ -3104,7 +3104,7 @@ static void ResetSlotLSNEndRecovery(StringInfo slotname) ReplicationSlotsComputeRequiredLSN(NULL); } -void inline XlogUpMinRecovPointSwitchChk(XLogRecPtr lsn) +inline void XlogUpMinRecovPointSwitchChk(XLogRecPtr lsn) { if (g_instance.comm_cxt.predo_cxt.pre_enable_switch != g_instance.attr.attr_storage.enable_update_max_page_flush_lsn) { @@ -12804,7 +12804,7 @@ static XLogSegNo CalcRecycleSegNoForHadrMainStandby(XLogRecPtr curFlush, XLogSeg XLByteToSeg(minRequired, slotSegNo); - if (slotSegNo <= 0) { + if (slotSegNo == 0) { /* segno = 1 show all file should be keep */ SegNoCanRecycled = 1; ereport(LOG, (errmsg("main standby keep all the xlog segments, because the minimal replication slot segno " diff --git a/src/gausskernel/storage/lmgr/lock.cpp b/src/gausskernel/storage/lmgr/lock.cpp index fa3a36130..c0294edef 100644 --- a/src/gausskernel/storage/lmgr/lock.cpp +++ b/src/gausskernel/storage/lmgr/lock.cpp @@ -1260,7 +1260,7 @@ static void RemoveLocalLock(LOCALLOCK *locallock) } } -bool inline IsInSameLockGroup(const PROCLOCK *proclock1, const PROCLOCK *proclock2) +inline bool IsInSameLockGroup(const PROCLOCK *proclock1, const PROCLOCK *proclock2) { Assert(proclock1->groupLeader != t_thrd.proc || t_thrd.proc->lockGroupLeader != NULL); return proclock1 != proclock2 && proclock1->groupLeader == proclock2->groupLeader; diff --git a/src/gausskernel/storage/tcap/tcap_manager.cpp b/src/gausskernel/storage/tcap/tcap_manager.cpp index 2d2752bb1..213b64a83 100644 --- a/src/gausskernel/storage/tcap/tcap_manager.cpp +++ b/src/gausskernel/storage/tcap/tcap_manager.cpp @@ -1696,7 +1696,7 @@ static bool TrObjInRecyclebin(const ObjectAddress *obj) sd = systable_beginscan(rbRel, RecyclebinDbidRelidIndexId, true, NULL, 2, skey); while ((tup = systable_getnext(sd)) != NULL) { Form_pg_recyclebin rbForm = (Form_pg_recyclebin)GETSTRUCT(tup); - if ((TrObjType)rbForm->rcyoperation == 'd') { + if (rbForm->rcyoperation == 'd') { found = true; break; } From 0ad9497335efd0c2625e24054f4c6a33caaed6d2 Mon Sep 17 00:00:00 2001 From: duzhuolin Date: Tue, 8 Aug 2023 15:25:06 +0800 Subject: [PATCH 112/304] =?UTF-8?q?=E5=AE=89=E5=85=A8=E5=87=BD=E6=95=B0?= =?UTF-8?q?=E5=92=8C=E6=8C=87=E9=92=88=E8=B5=8B=E5=80=BC=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/parser/parse_utilcmd.cpp | 2 +- src/common/backend/utils/adt/timestamp.cpp | 4 +++- src/gausskernel/storage/mot/core/system/mot_error.cpp | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/common/backend/parser/parse_utilcmd.cpp b/src/common/backend/parser/parse_utilcmd.cpp index 3747f843b..7c3ac0e17 100644 --- a/src/common/backend/parser/parse_utilcmd.cpp +++ b/src/common/backend/parser/parse_utilcmd.cpp @@ -3706,7 +3706,7 @@ static IndexStmt* transformIndexConstraint(Constraint* constraint, CreateStmtCon } else if (attnum == 0) { // expresional index Node *indexkey = NULL; - attform = 0; + attform = NULL; if (indexpr_item == NULL) { ereport(ERROR, diff --git a/src/common/backend/utils/adt/timestamp.cpp b/src/common/backend/utils/adt/timestamp.cpp index 168ba556e..9f5bdcbf2 100644 --- a/src/common/backend/utils/adt/timestamp.cpp +++ b/src/common/backend/utils/adt/timestamp.cpp @@ -4916,6 +4916,7 @@ Datum generate_series_timestamptz(PG_FUNCTION_ARGS) Interval* step = PG_GETARG_INTERVAL_P(2); MemoryContext oldcontext; Interval interval_zero; + errno_t rc; /* create a function context for cross-call persistence */ funcctx = SRF_FIRSTCALL_INIT(); @@ -4937,7 +4938,8 @@ Datum generate_series_timestamptz(PG_FUNCTION_ARGS) fctx->step = *step; /* Determine sign of the interval */ - MemSet(&interval_zero, 0, sizeof(Interval)); + rc = memset_s(&interval_zero, sizeof(Interval), 0, sizeof(Interval)); + securec_check(rc, "\0", "\0"); fctx->step_sign = interval_cmp_internal(&fctx->step, &interval_zero); if (fctx->step_sign == 0) diff --git a/src/gausskernel/storage/mot/core/system/mot_error.cpp b/src/gausskernel/storage/mot/core/system/mot_error.cpp index 3f24413c1..c11265b44 100644 --- a/src/gausskernel/storage/mot/core/system/mot_error.cpp +++ b/src/gausskernel/storage/mot/core/system/mot_error.cpp @@ -161,7 +161,7 @@ extern void PushErrorV(int errorCode, int severity, const char* file, int line, va_list args2; va_copy(args2, args); errno_t erc = vsnprintf_truncated_s(errorFrame->m_errorMessage, MOT_MAX_ERROR_MESSAGE, format, args2); - securec_check_ss(erc, "\0", "\0"); + securec_check_ss_c(erc, "\0", "\0"); errorFrame->m_errorMessage[MOT_MAX_ERROR_MESSAGE - 1] = 0; ++errorFrameCount; } From 460b37d1cf41000d72af8d14f50aed00faf71b00 Mon Sep 17 00:00:00 2001 From: duzhuolin Date: Tue, 8 Aug 2023 07:33:43 +0000 Subject: [PATCH 113/304] update src/gausskernel/storage/mot/core/system/mot_error.cpp. Signed-off-by: duzhuolin --- src/gausskernel/storage/mot/core/system/mot_error.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gausskernel/storage/mot/core/system/mot_error.cpp b/src/gausskernel/storage/mot/core/system/mot_error.cpp index c11265b44..3f24413c1 100644 --- a/src/gausskernel/storage/mot/core/system/mot_error.cpp +++ b/src/gausskernel/storage/mot/core/system/mot_error.cpp @@ -161,7 +161,7 @@ extern void PushErrorV(int errorCode, int severity, const char* file, int line, va_list args2; va_copy(args2, args); errno_t erc = vsnprintf_truncated_s(errorFrame->m_errorMessage, MOT_MAX_ERROR_MESSAGE, format, args2); - securec_check_ss_c(erc, "\0", "\0"); + securec_check_ss(erc, "\0", "\0"); errorFrame->m_errorMessage[MOT_MAX_ERROR_MESSAGE - 1] = 0; ++errorFrameCount; } From 1db23f5dd0ecb720ac165d6c27b76f739d499ef6 Mon Sep 17 00:00:00 2001 From: chenxiaobin19 <1025221611@qq.com> Date: Tue, 8 Aug 2023 17:09:51 +0800 Subject: [PATCH 114/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=B9=B6=E8=A1=8C?= =?UTF-8?q?=E5=9B=9E=E6=94=BE=E4=B8=8Bpitr=E5=90=AF=E5=8A=A8=E5=A4=B1?= =?UTF-8?q?=E8=B4=A5=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/storage/access/transam/xlog.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/gausskernel/storage/access/transam/xlog.cpp b/src/gausskernel/storage/access/transam/xlog.cpp index 547e4850d..80202ef1a 100755 --- a/src/gausskernel/storage/access/transam/xlog.cpp +++ b/src/gausskernel/storage/access/transam/xlog.cpp @@ -7173,6 +7173,7 @@ static void RemoveNonParentXlogFiles(XLogRecPtr switchpoint, TimeLineID newTLI) void CheckBeforeTruncateXLog(XLogRecPtr endRecPtr) { const uint32 shiftSize = 32; + ProcTxnWorkLoad(true); if (XLogRecPtrIsInvalid(t_thrd.xlog_cxt.minRecoveryPoint) || XLByteLT(endRecPtr, t_thrd.xlog_cxt.minRecoveryPoint) || From d7a6dd8aa44d8e61785ef39289b3c80d365d6639 Mon Sep 17 00:00:00 2001 From: dongning12 Date: Thu, 3 Aug 2023 10:55:34 +0800 Subject: [PATCH 115/304] =?UTF-8?q?[=E8=B5=84=E6=BA=90=E6=B1=A0=E5=8C=96]o?= =?UTF-8?q?ldest=5Fxmin=E6=8E=A8=E8=BF=9B=E9=87=8D=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/ddes/adapter/Makefile | 3 +- src/gausskernel/ddes/adapter/ss_dms.cpp | 6 + .../ddes/adapter/ss_dms_auxiliary.cpp | 141 +++++++++++ .../ddes/adapter/ss_dms_callback.cpp | 42 +++- src/gausskernel/ddes/adapter/ss_init.cpp | 3 + .../ddes/adapter/ss_transaction.cpp | 43 +--- src/gausskernel/ddes/adapter/ss_xmin.cpp | 236 ++++++++++++++++++ src/gausskernel/ddes/ddes_commit_id | 2 +- .../process/postmaster/postmaster.cpp | 11 + .../process/threadpool/knl_instance.cpp | 19 +- src/gausskernel/storage/ipc/procarray.cpp | 20 +- src/include/ddes/dms/dms_api.h | 31 ++- src/include/ddes/dms/ss_common_attr.h | 7 +- src/include/ddes/dms/ss_dms.h | 3 + src/include/ddes/dms/ss_dms_auxiliary.h | 31 +++ src/include/ddes/dms/ss_dms_recovery.h | 1 + src/include/ddes/dms/ss_xmin.h | 68 +++++ src/include/gs_thread.h | 1 + src/include/knl/knl_instance.h | 4 +- src/include/knl/knl_thread.h | 5 + src/include/miscadmin.h | 1 + src/test/ss/ha_test.sh | 2 +- 22 files changed, 600 insertions(+), 80 deletions(-) create mode 100644 src/gausskernel/ddes/adapter/ss_dms_auxiliary.cpp create mode 100644 src/gausskernel/ddes/adapter/ss_xmin.cpp create mode 100644 src/include/ddes/dms/ss_dms_auxiliary.h create mode 100644 src/include/ddes/dms/ss_xmin.h diff --git a/src/gausskernel/ddes/adapter/Makefile b/src/gausskernel/ddes/adapter/Makefile index 121134277..a12f0c797 100644 --- a/src/gausskernel/ddes/adapter/Makefile +++ b/src/gausskernel/ddes/adapter/Makefile @@ -22,7 +22,8 @@ ifneq "$(MAKECMDGOALS)" "clean" endif OBJS = ss_dms_bufmgr.o ss_dms_callback.o ss_dms_recovery.o ss_dms.o ss_init.o \ - ss_reform_common.o ss_switchover.o ss_transaction.o ss_aio.o ss_txnstatus.o + ss_reform_common.o ss_switchover.o ss_transaction.o ss_aio.o ss_txnstatus.o \ + ss_xmin.o ss_dms_auxiliary.o include $(top_srcdir)/src/gausskernel/common.mk diff --git a/src/gausskernel/ddes/adapter/ss_dms.cpp b/src/gausskernel/ddes/adapter/ss_dms.cpp index 251d6e725..ee759b06d 100644 --- a/src/gausskernel/ddes/adapter/ss_dms.cpp +++ b/src/gausskernel/ddes/adapter/ss_dms.cpp @@ -128,6 +128,7 @@ int ss_dms_func_init() SS_RETURN_IFERR(DMS_LOAD_SYMBOL_FUNC(dms_validate_drc)); SS_RETURN_IFERR(DMS_LOAD_SYMBOL_FUNC(dms_reform_req_opengauss_ondemand_redo_buffer)); SS_RETURN_IFERR(DMS_LOAD_SYMBOL_FUNC(dms_get_mes_max_watting_rooms)); + SS_RETURN_IFERR(DMS_LOAD_SYMBOL_FUNC(dms_send_opengauss_oldest_xmin)); g_ss_dms_func.inited = true; return DMS_SUCCESS; } @@ -352,3 +353,8 @@ unsigned int dms_get_mes_max_watting_rooms(void) { return g_ss_dms_func.dms_get_mes_max_watting_rooms(); } + +int dms_send_opengauss_oldest_xmin(dms_context_t *dms_ctx, unsigned long long oldest_xmin, unsigned char dest_id) +{ + return g_ss_dms_func.dms_send_opengauss_oldest_xmin(dms_ctx, oldest_xmin, dest_id); +} \ No newline at end of file diff --git a/src/gausskernel/ddes/adapter/ss_dms_auxiliary.cpp b/src/gausskernel/ddes/adapter/ss_dms_auxiliary.cpp new file mode 100644 index 000000000..7f387a9d1 --- /dev/null +++ b/src/gausskernel/ddes/adapter/ss_dms_auxiliary.cpp @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * --------------------------------------------------------------------------------------- + * + * ss_dms_auxiliary.cpp + * dms auxiliary related + * + * + * IDENTIFICATION + * src/gausskernel/ddes/adapter/ss_dms_auxiliary.cpp + * + * --------------------------------------------------------------------------------------- + */ + +#include "ddes/dms/ss_dms_auxiliary.h" +#include "ddes/dms/ss_xmin.h" +#include "postgres.h" +#include "storage/procarray.h" +#include "storage/ipc.h" + +#define DMS_AUXILIARY_SLEEP_TIME (1000) // 1s 1000ms +static void dms_auxiliary_request_shutdown_handler(SIGNAL_ARGS) +{ + int save_errno = errno; + + t_thrd.dms_aux_cxt.shutdown_requested = true; + if (t_thrd.proc) { + SetLatch(&t_thrd.proc->procLatch); + } + errno = save_errno; +} + +static void SetupDmsAuxiliarySignalHook(void) +{ + /* + * Reset some signals that are accepted by postmaster but not here + */ + (void)gspqsignal(SIGHUP, SIG_IGN); + (void)gspqsignal(SIGINT, dms_auxiliary_request_shutdown_handler); + (void)gspqsignal(SIGTERM, dms_auxiliary_request_shutdown_handler); + (void)gspqsignal(SIGQUIT, dms_auxiliary_request_shutdown_handler); /* hard crash time */ + (void)gspqsignal(SIGALRM, SIG_IGN); + (void)gspqsignal(SIGPIPE, SIG_IGN); + (void)gspqsignal(SIGUSR1, SIG_IGN); + (void)gspqsignal(SIGUSR2, SIG_IGN); + (void)gspqsignal(SIGURG, print_stack); + /* + * Reset some signals that are accepted by postmaster but not here + */ + (void)gspqsignal(SIGCHLD, SIG_DFL); + (void)gspqsignal(SIGTTIN, SIG_DFL); + (void)gspqsignal(SIGTTOU, SIG_DFL); + (void)gspqsignal(SIGCONT, SIG_DFL); + (void)gspqsignal(SIGWINCH, SIG_DFL); +} + +void dms_auxiliary_handle_exception() +{ + /* Since not using PG_TRY, must reset error stack by hand */ + t_thrd.log_cxt.error_context_stack = NULL; + t_thrd.log_cxt.call_stack = NULL; + + /* Prevent interrupts while cleaning up */ + HOLD_INTERRUPTS(); + + /* Report the error to the server log */ + EmitErrorReport(); + + /* Now we can allow interrupts again */ + RESUME_INTERRUPTS(); + + /* + * Sleep at least 1 second after any error. A write error is likely + * to be repeated, and we don't want to be filling the error logs as + * fast as we can. + */ + pg_usleep(1000000L); + return; +} + +void DmsAuxiliaryMain(void) +{ + sigjmp_buf localSigjmpBuf; + t_thrd.role = DMS_AUXILIARY_THREAD; + SetupDmsAuxiliarySignalHook(); + (void)sigdelset(&t_thrd.libpq_cxt.BlockSig, SIGQUIT); + + if (sigsetjmp(localSigjmpBuf, 1) != 0) { + ereport(WARNING, (errmodule(MOD_DMS), errmsg("dms auxiliary thread exception occured."))); + dms_auxiliary_handle_exception(); + } + + /* We can now handle ereport(ERROR)*/ + t_thrd.log_cxt.PG_exception_stack = &localSigjmpBuf; + + gs_signal_setmask(&t_thrd.libpq_cxt.UnBlockSig, NULL); + (void)gs_signal_unblock_sigusr2(); + + HASHCTL ctl; + errno_t rc = memset_s(&ctl, sizeof(ctl), 0, sizeof(ctl)); + securec_check(rc, "\0", "\0"); + ctl.keysize = sizeof(ss_snap_xmin_key_t); + ctl.entrysize = sizeof(ss_snap_xmin_item_t); + ctl.hash = tag_hash; + ss_xmin_info_t *xmin_info = &g_instance.dms_cxt.SSXminInfo; + xmin_info->snap_cache = HeapMemInitHash("DMS snap xmin cache", 60, 30000, &ctl, HASH_ELEM | HASH_FUNCTION); + + for (;;) { + if (t_thrd.dms_aux_cxt.shutdown_requested) { + u_sess->attr.attr_common.ExitOnAnyError = true; + proc_exit(0); + } + + if (g_instance.dms_cxt.dms_status < DMS_STATUS_IN || SS_IN_REFORM) { + pg_usleep(SS_REFORM_WAIT_TIME); + continue; + } + + if (SS_NORMAL_PRIMARY) { + MaintXminInPrimary(); + } else if (SS_NORMAL_STANDBY) { + MaintXminInStandby(); + } + + int rc = WaitLatch(&t_thrd.proc->procLatch, WL_TIMEOUT | WL_POSTMASTER_DEATH, DMS_AUXILIARY_SLEEP_TIME); + if (rc & WL_POSTMASTER_DEATH) { + gs_thread_exit(1); + } + } +} \ No newline at end of file diff --git a/src/gausskernel/ddes/adapter/ss_dms_callback.cpp b/src/gausskernel/ddes/adapter/ss_dms_callback.cpp index 14c7aa0bf..6ebada131 100644 --- a/src/gausskernel/ddes/adapter/ss_dms_callback.cpp +++ b/src/gausskernel/ddes/adapter/ss_dms_callback.cpp @@ -197,7 +197,7 @@ static int CBGetTxnCSN(void *db_handle, dms_opengauss_xid_csn_t *csn_req, dms_op return ret; } -static int CBGetSnapshotData(void *db_handle, dms_opengauss_txn_snapshot_t *txn_snapshot) +static int CBGetSnapshotData(void *db_handle, dms_opengauss_txn_snapshot_t *txn_snapshot, uint8 inst_id) { /* SS_STANDBY_CLUSTER_NORMAL_MAIN_STANDBY always is in recovery progress, but it can acquire snapshot*/ if (RecoveryInProgress() && !SS_STANDBY_CLUSTER_NORMAL_MAIN_STANDBY) { @@ -216,7 +216,11 @@ static int CBGetSnapshotData(void *db_handle, dms_opengauss_txn_snapshot_t *txn_ txn_snapshot->xmax = snapshot.xmax; txn_snapshot->snapshotcsn = snapshot.snapshotcsn; txn_snapshot->localxmin = u_sess->utils_cxt.RecentGlobalXmin; - retCode = DMS_SUCCESS; + if (RecordSnapshotBeforeSend(inst_id, txn_snapshot->xmin)) { + retCode = DMS_SUCCESS; + } else { + retCode = DMS_ERROR; + } } } PG_CATCH(); @@ -460,6 +464,8 @@ static int CBSaveStableList(void *db_handle, unsigned long long list_stable, uns g_instance.dms_cxt.SSReformerControl.list_stable = list_stable; int ret = DMS_ERROR; SSLockReleaseAll(); + SSSyncOldestXminWhenReform(reformer_id); + if ((int)primary_id == SS_MY_INST_ID) { if (g_instance.dms_cxt.SSClusterState > NODESTATE_NORMAL) { Assert(g_instance.dms_cxt.SSClusterState == NODESTATE_STANDBY_PROMOTED || @@ -1022,9 +1028,6 @@ static int32 CBProcessBroadcast(void *db_handle, char *data, unsigned int len, c PG_TRY(); { switch (bcast_op) { - case BCAST_GET_XMIN: - ret = SSGetOldestXmin(data, len, output_msg, output_msg_len); - break; case BCAST_SI: ret = SSProcessSharedInvalMsg(data, len); break; @@ -1079,9 +1082,6 @@ static int32 CBProcessBroadcastAck(void *db_handle, char *data, unsigned int len SSBroadcastOpAck bcast_op = *(SSBroadcastOpAck *)data; switch (bcast_op) { - case BCAST_GET_XMIN_ACK: - ret = SSGetOldestXminAck((SSBroadcastXminAck *)data); - break; case BCAST_CHECK_DB_BACKENDS_ACK: ret = SSCheckDbBackendsAck(data, len); break; @@ -1678,7 +1678,21 @@ static void ReformTypeToString(SSReformType reform_type, char* ret_str) return; } -static void CBReformStartNotify(void *db_handle, dms_role_t role, unsigned char reform_type) +static void SSXminInfoPrepare() +{ + ss_xmin_info_t *xmin_info = &g_instance.dms_cxt.SSXminInfo; + if (g_instance.dms_cxt.SSReformInfo.dms_role == DMS_ROLE_REFORMER) { + xmin_info->global_oldest_xmin_active = false; + for (int i = 0; i < DMS_MAX_INSTANCES; i++) { + xmin_info->node_table[i].active = false; + xmin_info->node_table[i].notify_oldest_xmin = MaxTransactionId; + } + } + xmin_info->bitmap_active_nodes = 0; +} + +static void CBReformStartNotify(void *db_handle, dms_role_t role, unsigned char reform_type, + unsigned long long bitmap_nodes) { ss_reform_info_t *reform_info = &g_instance.dms_cxt.SSReformInfo; reform_info->reform_type = (SSReformType)reform_type; @@ -1707,8 +1721,11 @@ static void CBReformStartNotify(void *db_handle, dms_role_t role, unsigned char ereport(LOG, (errmodule(MOD_DMS), errmsg("[SS failover] failover trigger."))); } } + + reform_info->bitmap_nodes = bitmap_nodes; reform_info->dms_role = role; reform_info->in_reform = true; + SSXminInfoPrepare(); char reform_type_str[reform_type_str_len] = {0}; ReformTypeToString(reform_info->reform_type, reform_type_str); @@ -1826,6 +1843,12 @@ static int CBMarkNeedFlush(void *db_handle, char *pageid) return DMS_SUCCESS; } +static int CBUpdateNodeOldestXmin(void *db_handle, uint8 inst_id, unsigned long long oldest_xmin) +{ + SSUpdateNodeOldestXmin(inst_id, oldest_xmin); + return GS_SUCCESS; +} + void DmsCallbackThreadShmemInit(unsigned char need_startup, char **reg_data) { /* in dorado mode, we need to wait sharestorageinit finished */ @@ -1997,4 +2020,5 @@ void DmsInitCallback(dms_callback_t *callback) callback->db_check_lock = CBDBCheckLock; callback->cache_msg = CBCacheMsg; callback->need_flush = CBMarkNeedFlush; + callback->update_node_oldest_xmin = CBUpdateNodeOldestXmin; } diff --git a/src/gausskernel/ddes/adapter/ss_init.cpp b/src/gausskernel/ddes/adapter/ss_init.cpp index a4f862cbc..4c84acae2 100644 --- a/src/gausskernel/ddes/adapter/ss_init.cpp +++ b/src/gausskernel/ddes/adapter/ss_init.cpp @@ -503,6 +503,9 @@ void DMSUninit() return; } + ereport(LOG, (errmsg("dms xmin maintainer thread exit"))); + signal_child(g_instance.pid_cxt.DmsAuxiliaryPID, SIGTERM, -1); + g_instance.dms_cxt.dmsInited = false; ereport(LOG, (errmsg("DMS uninit worker threads, DRC, errdesc and DL"))); dms_uninit(); diff --git a/src/gausskernel/ddes/adapter/ss_transaction.cpp b/src/gausskernel/ddes/adapter/ss_transaction.cpp index 3561edf1d..77e2ed168 100644 --- a/src/gausskernel/ddes/adapter/ss_transaction.cpp +++ b/src/gausskernel/ddes/adapter/ss_transaction.cpp @@ -384,47 +384,6 @@ TransactionId SSMultiXactIdGetUpdateXid(TransactionId xmax, uint16 t_infomask, u return update_xid; } -int SSGetOldestXmin(char *data, uint32 len, char *output_msg, uint32 *output_msg_len) -{ - if (unlikely(len != sizeof(SSBroadcastXmin))) { - ereport(DEBUG1, (errmsg("invalid broadcast xmin message"))); - return DMS_ERROR; - } - - SSBroadcastXminAck* getXminReq = (SSBroadcastXminAck *)output_msg; - getXminReq->type = BCAST_GET_XMIN_ACK; - GetOldestGlobalProcXmin(&(getXminReq->xmin)); - *output_msg_len = sizeof(SSBroadcastXminAck); - return DMS_SUCCESS; -} - -/* Calbulate the oldest xmin during broadcast xmin ack */ -int SSGetOldestXminAck(SSBroadcastXminAck *ack_data) -{ - TransactionId xmin_ack = pg_atomic_read_u64(&g_instance.dms_cxt.xminAck); - if (TransactionIdIsValid(ack_data->xmin) && TransactionIdIsNormal(ack_data->xmin) && - TransactionIdPrecedes(ack_data->xmin, xmin_ack)) { - pg_atomic_write_u64(&g_instance.dms_cxt.xminAck, ack_data->xmin); - } - return DMS_SUCCESS; -} - -bool SSGetOldestXminFromAllStandby() -{ - dms_context_t dms_ctx; - InitDmsContext(&dms_ctx); - SSBroadcastXmin xmin_data; - xmin_data.type = BCAST_GET_XMIN; - xmin_data.xmin = InvalidTransactionId; - pg_atomic_write_u64(&g_instance.dms_cxt.xminAck, MaxTransactionId); - int ret = dms_broadcast_msg(&dms_ctx, (char *)&xmin_data, sizeof(SSBroadcastXmin), - (unsigned char)true, SS_BROADCAST_WAIT_FIVE_SECONDS); - if (ret != DMS_SUCCESS) { - return false; - } - return true; -} - void SSIsPageHitDms(RelFileNode& node, BlockNumber page, int pagesNum, uint64 *pageMap, int *bitCount) { dms_context_t dms_ctx; @@ -874,4 +833,4 @@ void SSStandbyUpdateRedirectInfo() redirect_manager->ss_standby_sxid = dms_sw_info.sxid; redirect_manager->ss_standby_scid = dms_sw_info.scid; -} +} \ No newline at end of file diff --git a/src/gausskernel/ddes/adapter/ss_xmin.cpp b/src/gausskernel/ddes/adapter/ss_xmin.cpp new file mode 100644 index 000000000..5decb60ad --- /dev/null +++ b/src/gausskernel/ddes/adapter/ss_xmin.cpp @@ -0,0 +1,236 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * --------------------------------------------------------------------------------------- + * + * ss_xmin.cpp + * ss xmin related + * + * + * IDENTIFICATION + * src/gausskernel/ddes/adapter/ss_xmin.cpp + * + * --------------------------------------------------------------------------------------- + */ + +#include "ddes/dms/ss_xmin.h" +#include "storage/procarray.h" +#include "storage/ipc.h" +#include "ddes/dms/ss_dms_bufmgr.h" +#include "ddes/dms/ss_dms.h" +#include "ddes/dms/ss_common_attr.h" +#include "knl/knl_instance.h" + +extern void CalculateLocalLatestSnapshot(bool forceCalc); + +uint64 GetOldestXminInNodeTable() +{ + ss_xmin_info_t *xmin_info = &g_instance.dms_cxt.SSXminInfo; + uint64 min_xmin = MaxTransactionId; + for (int i = 0; i < DMS_MAX_INSTANCES; i++) { + if ((uint64)(1 << i) > g_instance.dms_cxt.SSXminInfo.bitmap_active_nodes) { + break; + } + + if (!xmin_info->node_table[i].active) { + continue; + } + + ss_node_xmin_item_t *item = &xmin_info->node_table[i]; + SpinLockAcquire(&item->item_lock); + if (TransactionIdPrecedes(item->notify_oldest_xmin, min_xmin)) { + min_xmin = item->notify_oldest_xmin; + } + SpinLockRelease(&item->item_lock); + } + return min_xmin; +} + +void MaintXminInPrimary(void) +{ + if (SS_IN_REFORM) { + pg_usleep(SS_REFORM_WAIT_TIME); + return; + } + ss_xmin_info_t *xmin_info = &g_instance.dms_cxt.SSXminInfo; + struct HTAB* snap_cache = g_instance.dms_cxt.SSXminInfo.snap_cache; + HASH_SEQ_STATUS hash_seq; + hash_seq_init(&hash_seq, snap_cache); + ss_snap_xmin_item_t *xmin_item; + uint64 snap_xmin = MaxTransactionId; + TimestampTz cur_time = GetCurrentTimestamp(); + + while ((xmin_item = (ss_snap_xmin_item_t*)hash_seq_search(&hash_seq)) != NULL) { + if (SS_IN_REFORM) { + return; + } + + if (TimestampDifferenceExceeds(xmin_item->timestamp, cur_time, DMS_MSG_MAX_WAIT_TIME)) { + ss_snap_xmin_key_t key{.xmin = xmin_item->xmin}; + hash_search(snap_cache, &key, HASH_REMOVE, NULL); + continue; + } + + if (TransactionIdPrecedes(xmin_item->xmin, snap_xmin)) { + snap_xmin = xmin_item->xmin; + } + } + + if (snap_xmin == MaxTransactionId) { + TimestampTz recent_time = pg_atomic_read_u64((volatile uint64*)&xmin_info->recent_snap_send_time); + if (TimestampDifferenceExceeds(cur_time, recent_time, 0)) { + return; + } + } + + xmin_info->snap_oldest_xmin = snap_xmin; + uint64 new_global_xmin = GetOldestXminInNodeTable(); + if (TransactionIdPrecedes(snap_xmin, new_global_xmin)) { + new_global_xmin = snap_xmin; + } + + SpinLockAcquire(&xmin_info->global_oldest_xmin_lock); + xmin_info->global_oldest_xmin = new_global_xmin; + SpinLockRelease(&xmin_info->global_oldest_xmin_lock); +} + +void MaintXminInStandby(void) +{ + if (SS_IN_REFORM) { + pg_usleep(SS_REFORM_WAIT_TIME); + return; + } + uint64 oldest_xmin = MaxTransactionId; + GetOldestGlobalProcXmin(&oldest_xmin); + dms_context_t dms_cxt = {0}; + InitDmsContext(&dms_cxt); + dms_send_opengauss_oldest_xmin(&dms_cxt, oldest_xmin, SS_PRIMARY_ID); +} + +bool RecordSnapshotBeforeSend(uint8 inst_id, uint64 xmin) +{ + ss_xmin_info_t *xmin_info = &g_instance.dms_cxt.SSXminInfo; + struct HTAB* snap_cache = xmin_info->snap_cache; + if (snap_cache == NULL) { + ereport(WARNING, (errmodule(MOD_DMS), errmsg("snap_cache is not ready, please retry"))); + return false; + } + + ss_snap_xmin_key_t key = {.xmin = xmin}; + ss_snap_xmin_item_t *xmin_item = (ss_snap_xmin_item_t*)hash_search(snap_cache, &key, HASH_ENTER_NULL, NULL); + if (xmin_item == NULL) { + ereport(WARNING, (errmodule(MOD_DMS), errmsg("insert snapshot into snap_cache table failed, " + "capacity is not enough"))); + return false; + } + xmin_item->xmin = xmin; + TimestampTz send_time = GetCurrentTimestamp(); + bool ret = false; + do { + uint64 recent_time = pg_atomic_read_u64((volatile uint64*)&xmin_info->recent_snap_send_time); + if ((uint64)send_time <= recent_time) { + break; + } + ret = pg_atomic_compare_exchange_u64((volatile uint64*)&xmin_info->recent_snap_send_time, &recent_time, send_time); + } while (!ret); + xmin_item->timestamp = send_time; + return true; +} + +uint64 SSGetGlobalOldestXmin(uint64 globalxmin) +{ + ss_xmin_info_t *xmin_info = &g_instance.dms_cxt.SSXminInfo; + if (!xmin_info->global_oldest_xmin_active) { + return globalxmin; + } + + uint64 ret_globalxmin = globalxmin; + SpinLockAcquire(&xmin_info->global_oldest_xmin_lock); + if (TransactionIdPrecedes(xmin_info->global_oldest_xmin, globalxmin)) { + ret_globalxmin = xmin_info->global_oldest_xmin; + } + SpinLockRelease(&xmin_info->global_oldest_xmin_lock); + return ret_globalxmin; +} + +void SSUpdateNodeOldestXmin(uint8 inst_id, unsigned long long oldest_xmin) +{ + if (!TransactionIdIsNormal(oldest_xmin)) { + return; + } + + ss_xmin_info_t *xmin_info = &g_instance.dms_cxt.SSXminInfo; + ss_reform_info_t *reform_info = &g_instance.dms_cxt.SSReformInfo; + + SpinLockAcquire(&xmin_info->node_table[inst_id].item_lock); + xmin_info->node_table[inst_id].notify_oldest_xmin = oldest_xmin; + SpinLockRelease(&xmin_info->node_table[inst_id].item_lock); + + if (SS_IN_REFORM) { + if ((reform_info->bitmap_nodes |= (1 << inst_id)) == 0) { + return; + } + + SpinLockAcquire(&xmin_info->bitmap_active_nodes_lock); + xmin_info->bitmap_active_nodes |= 1 << inst_id; + xmin_info->node_table[inst_id].active = true; + SpinLockRelease(&xmin_info->bitmap_active_nodes_lock); + + SpinLockAcquire(&xmin_info->global_oldest_xmin_lock); + if (TransactionIdPrecedes(oldest_xmin, xmin_info->global_oldest_xmin)) { + xmin_info->global_oldest_xmin = oldest_xmin; + } + SpinLockRelease(&xmin_info->global_oldest_xmin_lock); + + SpinLockAcquire(&xmin_info->bitmap_active_nodes_lock); + if (xmin_info->bitmap_active_nodes == reform_info->bitmap_nodes) { + xmin_info->global_oldest_xmin_active = true; + } + SpinLockRelease(&xmin_info->bitmap_active_nodes_lock); + } + return; +} + +void SSSyncOldestXminWhenReform(uint8 reformer_id) +{ + ss_xmin_info_t *xmin_info = &g_instance.dms_cxt.SSXminInfo; + ss_reform_info_t *reform_info = &g_instance.dms_cxt.SSReformInfo; + + if (reform_info->dms_role == DMS_ROLE_REFORMER) { + while (!xmin_info->global_oldest_xmin_active) { + if (dms_reform_failed()) { + break; + } + uint64 tmp_nodes = xmin_info->bitmap_active_nodes | (1 << SS_MY_INST_ID); + if (tmp_nodes == reform_info->bitmap_nodes) { + break; + } + pg_usleep(1000L); + } + LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE); + CalculateLocalLatestSnapshot(true); + LWLockRelease(ProcArrayLock); + } else { + int ret = DMS_SUCCESS; + do { + if (dms_reform_failed()) { + break; + } + uint64 oldest_xmin = MaxTransactionId; + GetOldestGlobalProcXmin(&oldest_xmin); + dms_context_t dms_cxt = {0}; + InitDmsContext(&dms_cxt); + ret = dms_send_opengauss_oldest_xmin(&dms_cxt, oldest_xmin, reformer_id); + } while (ret != DMS_SUCCESS); + } +} \ No newline at end of file diff --git a/src/gausskernel/ddes/ddes_commit_id b/src/gausskernel/ddes/ddes_commit_id index 98b81de24..d68e79777 100644 --- a/src/gausskernel/ddes/ddes_commit_id +++ b/src/gausskernel/ddes/ddes_commit_id @@ -1,2 +1,2 @@ -dms_commit_id=a9d399af041ae5491149fdb7c58e5115da3065b2 +dms_commit_id=b12ceed7bb0d3de049d8b4858180c77110ff6444 dss_commit_id=f9bb2fa60f02331b0373559776a8ebdba86c002f diff --git a/src/gausskernel/process/postmaster/postmaster.cpp b/src/gausskernel/process/postmaster/postmaster.cpp index 4e8110be0..1c60bfb6f 100644 --- a/src/gausskernel/process/postmaster/postmaster.cpp +++ b/src/gausskernel/process/postmaster/postmaster.cpp @@ -260,6 +260,7 @@ #include "storage/dss/dss_log.h" #include "ddes/dms/ss_switchover.h" #include "ddes/dms/ss_reform_common.h" +#include "ddes/dms/ss_dms_auxiliary.h" #ifdef ENABLE_UT #define static @@ -3070,6 +3071,7 @@ int PostmasterMain(int argc, char* argv[]) if (g_instance.attr.attr_storage.dms_attr.enable_dms) { /* need to initialize before STARTUP */ DMSInit(); + g_instance.pid_cxt.DmsAuxiliaryPID = initialize_util_thread(DMS_AUXILIARY_THREAD); } } @@ -13252,6 +13254,9 @@ static void SetAuxType() t_thrd.bootstrap_cxt.MyAuxProcType = TsCompactionAuxiliaryProcess; break; #endif /* ENABLE_MULTIPLE_NODES */ + case DMS_AUXILIARY_THREAD: + t_thrd.bootstrap_cxt.MyAuxProcType = DmsAuxiliaryProcess; + break; default: ereport(ERROR, (errmsg("unrecorgnized proc type %d", thread_role))); } @@ -13546,6 +13551,10 @@ int GaussDbAuxiliaryThreadMain(knl_thread_arg* arg) proc_exit(1); break; #endif /* ENABLE_MULTIPLE_NODES */ + case DMS_AUXILIARY_THREAD: + DmsAuxiliaryMain(); + proc_exit(1); + break; default: ereport(PANIC, (errmsg("unrecognized process type: %d", (int)t_thrd.bootstrap_cxt.MyAuxProcType))); proc_exit(1); @@ -13789,6 +13798,7 @@ int GaussDbThreadMain(knl_thread_arg* arg) #endif /* ENABLE_MULTIPLE_NODES */ case THREADPOOL_LISTENER: case THREADPOOL_SCHEDULER: + case DMS_AUXILIARY_THREAD: case UNDO_RECYCLER: { SetAuxType(); /* Restore basic shared memory pointers */ @@ -14332,6 +14342,7 @@ static ThreadMetaData GaussdbThreadGate[] = { { GaussDbThreadMain, APPLY_LAUNCHER, "applylauncher", "apply launcher" }, { GaussDbThreadMain, APPLY_WORKER, "applyworker", "apply worker" }, { GaussDbThreadMain, STACK_PERF_WORKER, "stack_perf", "stack perf worker" }, + { GaussDbThreadMain, DMS_AUXILIARY_THREAD, "dms_auxiliary", "maintenance xmin in dms" }, /* Keep the block in the end if it may be absent !!! */ #ifdef ENABLE_MULTIPLE_NODES diff --git a/src/gausskernel/process/threadpool/knl_instance.cpp b/src/gausskernel/process/threadpool/knl_instance.cpp index a3b02e86b..09b54dd36 100755 --- a/src/gausskernel/process/threadpool/knl_instance.cpp +++ b/src/gausskernel/process/threadpool/knl_instance.cpp @@ -174,7 +174,6 @@ static void knl_g_dms_init(knl_g_dms_context *dms_cxt) { Assert(dms_cxt != NULL); dms_cxt->dmsProcSid = 0; - dms_cxt->xminAck = 0; dms_cxt->SSReformerControl.list_stable = 0; dms_cxt->SSReformerControl.primaryInstId = -1; dms_cxt->SSReformInfo.in_reform = false; @@ -203,6 +202,24 @@ static void knl_g_dms_init(knl_g_dms_context *dms_cxt) dms_cxt->resetSyscache = false; dms_cxt->finishedRecoverOldPrimaryDWFile = false; dms_cxt->dw_init = false; + + { + ss_xmin_info_t *xmin_info = &g_instance.dms_cxt.SSXminInfo; + for (int i = 0; i < DMS_MAX_INSTANCES; i++) { + ss_node_xmin_item_t *item = &xmin_info->node_table[i]; + item->active = false; + SpinLockInit(&item->item_lock); + item->notify_oldest_xmin = MaxTransactionId; + } + xmin_info->snap_cache = NULL; + xmin_info->snap_oldest_xmin = MaxTransactionId; + xmin_info->recent_snap_send_time = 0; + SpinLockInit(&xmin_info->global_oldest_xmin_lock); + xmin_info->global_oldest_xmin = MaxTransactionId; + xmin_info->global_oldest_xmin_active = false; + SpinLockInit(&xmin_info->bitmap_active_nodes_lock); + xmin_info->bitmap_active_nodes = 0; + } } static void knl_g_tests_init(knl_g_tests_context* tests_cxt) diff --git a/src/gausskernel/storage/ipc/procarray.cpp b/src/gausskernel/storage/ipc/procarray.cpp index ab7216d0a..7511bcac8 100755 --- a/src/gausskernel/storage/ipc/procarray.cpp +++ b/src/gausskernel/storage/ipc/procarray.cpp @@ -2362,12 +2362,10 @@ Snapshot GetSnapshotData(Snapshot snapshot, bool force_local_snapshot) } /* Check whether there's a standby requiring an older xmin when dms is enabled. */ - if (ENABLE_DMS && SS_STANDBY_CLUSTER_NORMAL_MAIN_STANDBY && SSGetOldestXminFromAllStandby()) { - TransactionId ss_oldest_xmin = pg_atomic_read_u64(&g_instance.dms_cxt.xminAck); - if (TransactionIdIsValid(ss_oldest_xmin) && TransactionIdIsNormal(ss_oldest_xmin) && - TransactionIdPrecedes(ss_oldest_xmin, u_sess->utils_cxt.RecentGlobalXmin)) { - u_sess->utils_cxt.RecentGlobalXmin = ss_oldest_xmin; - } + if (ENABLE_DMS && SS_STANDBY_CLUSTER_NORMAL_MAIN_STANDBY) { + ss_xmin_info_t* xmin_info = &g_instance.dms_cxt.SSXminInfo; + uint64 global_xmin = SSGetGlobalOldestXmin(u_sess->utils_cxt.RecentGlobalXmin); + u_sess->utils_cxt.RecentGlobalXmin = global_xmin; } /* Non-catalog tables can be vacuumed if older than this xid */ @@ -5029,12 +5027,9 @@ void CalculateLocalLatestSnapshot(bool forceCalc) if (TransactionIdPrecedes(xmin, globalxmin)) globalxmin = xmin; - if (ENABLE_DMS && SS_PRIMARY_MODE && SSGetOldestXminFromAllStandby()) { - TransactionId ss_oldest_xmin = pg_atomic_read_u64(&g_instance.dms_cxt.xminAck); - if (TransactionIdIsValid(ss_oldest_xmin) && TransactionIdIsNormal(ss_oldest_xmin) && - TransactionIdPrecedes(ss_oldest_xmin, globalxmin)) { - globalxmin = ss_oldest_xmin; - } + if (ENABLE_DMS && SS_PRIMARY_MODE) { + SSUpdateNodeOldestXmin(SS_MY_INST_ID, globalxmin); + globalxmin = SSGetGlobalOldestXmin(globalxmin); } t_thrd.xact_cxt.ShmemVariableCache->xmin = xmin; @@ -5401,7 +5396,6 @@ void UpdateXLogMaxCSN(CommitSeqNo xlogCSN) void GetOldestGlobalProcXmin(TransactionId *globalProcXmin) { TransactionId globalxmin = MaxTransactionId; - *globalProcXmin = InvalidTransactionId; ProcArrayStruct *arrayP = g_instance.proc_array_idx; int *pgprocnos = arrayP->pgprocnos; int numProcs = arrayP->numProcs; diff --git a/src/include/ddes/dms/dms_api.h b/src/include/ddes/dms/dms_api.h index b0c26edcb..513e58f7b 100644 --- a/src/include/ddes/dms/dms_api.h +++ b/src/include/ddes/dms/dms_api.h @@ -359,6 +359,7 @@ typedef enum en_dms_cr_status { DMS_CR_TRY_READ = 0, DMS_CR_LOCAL_READ, DMS_CR_READ_PAGE, + DMS_CR_READ_EDP_PAGE, DMS_CR_CONSTRUCT, DMS_CR_PAGE_VISIBLE, DMS_CR_CHECK_MASTER, @@ -472,6 +473,7 @@ typedef enum en_dms_wait_event { DMS_EVT_LATCH_S_REMOTE, DMS_EVT_ONDEMAND_REDO, DMS_EVT_PAGE_STATUS_INFO, + DMS_EVT_OPENGAUSS_SEND_XMIN, DMS_EVT_COUNT, @@ -545,7 +547,8 @@ typedef int(*dms_file_orglsn_recovery)(void *db_handle, void *recovery_list, int typedef int(*dms_opengauss_startup)(void *db_handle); typedef int(*dms_opengauss_recovery_standby)(void *db_handle, int inst_id); typedef int(*dms_opengauss_recovery_primary)(void *db_handle, int inst_id); -typedef void(*dms_reform_start_notify)(void *db_handle, dms_role_t role, unsigned char reform_type); +typedef void(*dms_reform_start_notify)(void *db_handle, dms_role_t role, unsigned char reform_type, + unsigned long long bitmap_nodes); typedef int(*dms_undo_init)(void *db_handle, unsigned char inst_id); typedef int(*dms_tx_area_init)(void *db_handle, unsigned char inst_id); typedef int(*dms_tx_area_load)(void *db_handle, unsigned char inst_id); @@ -576,7 +579,7 @@ typedef void *(*dms_get_db_handle)(unsigned int *db_handle_index, dms_session_ty typedef void (*dms_release_db_handle)(void *db_handle); typedef void *(*dms_stack_push_cr_cursor)(void *db_handle); typedef void (*dms_stack_pop_cr_cursor)(void *db_handle); -typedef void(*dms_init_cr_cursor)(void *cr_cursor, char pageid[DMS_PAGEID_SIZE], char xid[DMS_XID_SIZE], +typedef int(*dms_init_cr_cursor)(void *cr_cursor, char pageid[DMS_PAGEID_SIZE], char xid[DMS_XID_SIZE], unsigned long long query_scn, unsigned int ssn); typedef void(*dms_init_index_cr_cursor)(void *cr_cursor, char pageid[DMS_PAGEID_SIZE], char xid[DMS_XID_SIZE], unsigned long long query_scn, unsigned int ssn, char entry[DMS_PAGEID_SIZE], char *index_profile); @@ -596,8 +599,8 @@ typedef void(*dms_get_entry_pageid_from_cr_cursor)(void *cr_cursor, char index_e typedef void(*dms_get_index_profile_from_cr_cursor)(void *cr_cursor, char index_profile[DMS_INDEX_PROFILE_SIZE]); typedef void(*dms_get_xid_from_cr_cursor)(void *cr_cursor, char xid[DMS_XID_SIZE]); typedef void(*dms_get_rowid_from_cr_cursor)(void *cr_cursor, char rowid[DMS_ROWID_SIZE]); -typedef int(*dms_read_page)(void *db_handle, dms_read_page_assist_t *assist, char **page_addr); -typedef void(*dms_leave_page)(void *db_handle, unsigned char changed); +typedef int(*dms_read_page)(void *db_handle, dms_read_page_assist_t *assist, char **page_addr, unsigned int *status); +typedef void(*dms_leave_page)(void *db_handle, unsigned char changed, unsigned int status); typedef char *(*dms_mem_alloc)(void *context, unsigned int size); typedef void(*dms_mem_free)(void *context, void *ptr); typedef void(*dms_mem_reset)(void *context); @@ -616,7 +619,8 @@ typedef int(*dms_get_opengauss_txn_status)(void *db_handle, unsigned long long x typedef int(*dms_opengauss_lock_buffer)(void *db_handle, int buffer, unsigned char lock_mode, unsigned char* curr_mode); typedef int(*dms_get_txn_snapshot)(void *db_handle, unsigned int xmap, dms_txn_snapshot_t *txn_snapshot); -typedef int(*dms_get_opengauss_txn_snapshot)(void *db_handle, dms_opengauss_txn_snapshot_t *txn_snapshot); +typedef int(*dms_get_opengauss_txn_snapshot)(void *db_handle, dms_opengauss_txn_snapshot_t *txn_snapshot, + unsigned char inst_id); typedef int(*dms_get_opengauss_txn_of_master)(void *db_handle, dms_opengauss_txn_sw_info_t *txn_swinfo); typedef int(*dms_get_opengauss_page_status)(void *db_handle, dms_opengauss_relfilenode_t *rnode, unsigned int page, int page_num, dms_opengauss_page_status_result_t *page_result); @@ -672,6 +676,12 @@ typedef void (*dms_verify_page)(dms_buf_ctrl_t *buf_ctrl, char *new_page); typedef int (*dms_drc_validate)(void *db_handle); typedef int (*dms_db_check_lock)(void *db_handle); typedef int (*dms_cache_msg)(void *db_handle, char* msg); +typedef void (*dms_ckpt_enque_one_page)(void *db_handle, dms_buf_ctrl_t *ctrl); +typedef int (*dms_set_remove_point)(void *db_handle, unsigned int node_id, void *curr_point); +typedef int (*dms_get_enable_checksum)(void *db_handle); +typedef unsigned int (*dms_calc_page_checksum)(void *db_handle, dms_buf_ctrl_t *ctrl, unsigned int page_size); +typedef int (*dms_verify_page_checksum)(void *db_handle, dms_buf_ctrl_t *ctrl, unsigned int page_size, int cks); +typedef int (*dms_update_node_oldest_xmin)(void *db_handle, unsigned char inst_id, unsigned long long oldest_xmin); typedef struct st_dms_callback { // used in reform @@ -688,7 +698,8 @@ typedef struct st_dms_callback { dms_recovery recovery; dms_dw_recovery dw_recovery; dms_df_recovery df_recovery; - dms_file_orglsn_recovery file_orglsn_recovery; + dms_file_orglsn_recovery file_orglsn_recovery_part1; + dms_file_orglsn_recovery file_orglsn_recovery_part2; dms_db_is_primary db_is_primary; dms_get_open_status get_open_status; dms_undo_init undo_init; @@ -804,6 +815,12 @@ typedef struct st_dms_callback { dms_drc_validate drc_validate; dms_db_check_lock db_check_lock; dms_cache_msg cache_msg; + dms_ckpt_enque_one_page ckpt_enque_one_page; + dms_set_remove_point set_remove_point; + dms_get_enable_checksum get_enable_checksum; + dms_calc_page_checksum calc_page_checksum; + dms_verify_page_checksum verify_page_checksum; + dms_update_node_oldest_xmin update_node_oldest_xmin; } dms_callback_t; typedef struct st_dms_instance_net_addr { @@ -879,7 +896,7 @@ typedef enum en_dms_info_id { #define DMS_LOCAL_MINOR_VER_WEIGHT 1000 #define DMS_LOCAL_MAJOR_VERSION 0 #define DMS_LOCAL_MINOR_VERSION 0 -#define DMS_LOCAL_VERSION 77 +#define DMS_LOCAL_VERSION 84 #ifdef __cplusplus } diff --git a/src/include/ddes/dms/ss_common_attr.h b/src/include/ddes/dms/ss_common_attr.h index fca52f775..55abc990e 100644 --- a/src/include/ddes/dms/ss_common_attr.h +++ b/src/include/ddes/dms/ss_common_attr.h @@ -187,10 +187,10 @@ #define SS_ACQUIRE_LOCK_RETRY_INTERVAL (50) // 50ms #define DMS_MSG_MAX_WAIT_TIME (10 * 1000) // 10s +#define SS_REFORM_WAIT_TIME (5000) // 5ms typedef enum SSBroadcastOp { - BCAST_GET_XMIN = 0, - BCAST_CANCEL_TRX_FOR_SWITCHOVER, + BCAST_CANCEL_TRX_FOR_SWITCHOVER = 0, BCAST_SI, BCAST_SEGDROPTL, BCAST_DROP_REL_ALL_BUFFER, @@ -206,8 +206,7 @@ typedef enum SSBroadcastOp { } SSBroadcastOp; typedef enum SSBroadcastOpAck { - BCAST_GET_XMIN_ACK = 0, - BCAST_CANCEL_TRX_ACK, + BCAST_CANCEL_TRX_ACK = 0, BCAST_CHECK_DB_BACKENDS_ACK, BCAST_ACK_END } SSBroadcastOpAck; diff --git a/src/include/ddes/dms/ss_dms.h b/src/include/ddes/dms/ss_dms.h index 37477f398..1bf0d8fda 100644 --- a/src/include/ddes/dms/ss_dms.h +++ b/src/include/ddes/dms/ss_dms.h @@ -84,6 +84,7 @@ typedef struct st_ss_dms_func { int (*dms_reform_req_opengauss_ondemand_redo_buffer)(dms_context_t *dms_ctx, void *block_key, unsigned int key_len, int *redo_status); unsigned int (*dms_get_mes_max_watting_rooms)(void); + int (*dms_send_opengauss_oldest_xmin)(dms_context_t *dms_ctx, unsigned long long oldest_xmin, unsigned char dest_id); } ss_dms_func_t; int ss_dms_func_init(); @@ -131,6 +132,8 @@ void dms_validate_drc(dms_context_t *dms_ctx, dms_buf_ctrl_t *ctrl, unsigned lon int dms_reform_req_opengauss_ondemand_redo_buffer(dms_context_t *dms_ctx, void *block_key, unsigned int key_len, int *redo_status); unsigned int dms_get_mes_max_watting_rooms(void); +int dms_send_opengauss_oldest_xmin(dms_context_t *dms_ctx, unsigned long long oldest_xmin, unsigned char dest_id); + #ifdef __cplusplus } #endif diff --git a/src/include/ddes/dms/ss_dms_auxiliary.h b/src/include/ddes/dms/ss_dms_auxiliary.h new file mode 100644 index 000000000..f0d9dbdfd --- /dev/null +++ b/src/include/ddes/dms/ss_dms_auxiliary.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * --------------------------------------------------------------------------------------- + * + * ss_dms_auxiliary.h + * include header file for ss dms auxiliary + * + * + * IDENTIFICATION + * src/include/ddes/dms/ss_dms_auxiliary.h + * + * --------------------------------------------------------------------------------------- + */ + +#ifndef __SS_DMS_AUXILIARY_H__ +#define __SS_DMS_AUXILIARY_H__ + +void DmsAuxiliaryMain(void); + +#endif \ No newline at end of file diff --git a/src/include/ddes/dms/ss_dms_recovery.h b/src/include/ddes/dms/ss_dms_recovery.h index 8645feef7..0c5dc6547 100644 --- a/src/include/ddes/dms/ss_dms_recovery.h +++ b/src/include/ddes/dms/ss_dms_recovery.h @@ -62,6 +62,7 @@ typedef struct st_reform_info { bool in_reform; dms_role_t dms_role; SSReformType reform_type; + unsigned long long bitmap_nodes; } ss_reform_info_t; typedef enum st_failover_ckpt_status { diff --git a/src/include/ddes/dms/ss_xmin.h b/src/include/ddes/dms/ss_xmin.h new file mode 100644 index 000000000..9e15b9e64 --- /dev/null +++ b/src/include/ddes/dms/ss_xmin.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * --------------------------------------------------------------------------------------- + * + * ss_xmin.h + * include header file for ss xmin + * + * + * IDENTIFICATION + * src/include/ddes/dms/ss_xmin.h + * + * --------------------------------------------------------------------------------------- + */ + +#ifndef __SS_XMIN_H__ +#define __SS_XMIN_H__ + +#include "postgres.h" +#include "storage/lock/s_lock.h" +#include "datatype/timestamp.h" +#include "ddes/dms/dms_api.h" + +typedef struct st_snap_xmin_key { + uint64 xmin; +} ss_snap_xmin_key_t; + +typedef struct st_ss_snap_xmin_item { + uint64 xmin; + TimestampTz timestamp; +} ss_snap_xmin_item_t; + +typedef struct st_ss_node_xmin_item { + slock_t item_lock; + uint64 notify_oldest_xmin; + bool active; +} ss_node_xmin_item_t; + +typedef struct st_ss_xmin_info { + ss_node_xmin_item_t node_table[DMS_MAX_INSTANCES]; + struct HTAB* snap_cache; + uint64 snap_oldest_xmin; + volatile TimestampTz recent_snap_send_time; + slock_t global_oldest_xmin_lock; + uint64 global_oldest_xmin; + bool global_oldest_xmin_active; + slock_t bitmap_active_nodes_lock; + uint64 bitmap_active_nodes; +} ss_xmin_info_t; + +void MaintXminInPrimary(void); +void MaintXminInStandby(void); +bool RecordSnapshotBeforeSend(uint8 inst_id, uint64 xmin); +uint64 SSGetGlobalOldestXmin(uint64 globalxmin); +void SSUpdateNodeOldestXmin(uint8 inst_id, unsigned long long oldest_xmin); +void SSSyncOldestXminWhenReform(uint8 reformer_id); + +#endif \ No newline at end of file diff --git a/src/include/gs_thread.h b/src/include/gs_thread.h index 6b566a6c2..b8caca6ae 100755 --- a/src/include/gs_thread.h +++ b/src/include/gs_thread.h @@ -123,6 +123,7 @@ typedef enum knl_thread_role { APPLY_LAUNCHER, APPLY_WORKER, STACK_PERF_WORKER, + DMS_AUXILIARY_THREAD, BARRIER_PREPARSE, TS_COMPACTION, TS_COMPACTION_CONSUMER, diff --git a/src/include/knl/knl_instance.h b/src/include/knl/knl_instance.h index 3b41f74c4..461b1e36c 100755 --- a/src/include/knl/knl_instance.h +++ b/src/include/knl/knl_instance.h @@ -73,6 +73,7 @@ #include "postmaster/barrier_creator.h" #include "pgxc/barrier.h" #include "ddes/dms/ss_dms_recovery.h" +#include "ddes/dms/ss_xmin.h" const int NUM_PERCENTILE_COUNT = 2; const int INIT_NUMA_ALLOC_COUNT = 32; @@ -217,6 +218,7 @@ typedef struct knl_g_pid_context { ThreadId ApplyLauncerPID; ThreadId StackPerfPID; ThreadId CfsShrinkerPID; + ThreadId DmsAuxiliaryPID; } knl_g_pid_context; typedef struct { @@ -1198,7 +1200,6 @@ typedef struct knl_g_datadir_context { typedef struct knl_g_dms_context { uint32 dmsProcSid; - uint64 xminAck; dms_status_t dms_status; ClusterNodeState SSClusterState; ss_reformer_ctrl_t SSReformerControl; // saved in disk; saved by primary @@ -1215,6 +1216,7 @@ typedef struct knl_g_dms_context { char dmsInstAddr[MAX_REPLNODE_NUM][DMS_MAX_IP_LEN]; char conninfo[MAXPGPATH]; ss_dfx_stats_t SSDFxStats; + ss_xmin_info_t SSXminInfo; } knl_g_dms_context; typedef struct knl_instance_context { diff --git a/src/include/knl/knl_thread.h b/src/include/knl/knl_thread.h index 78fec000f..a56f99cc9 100755 --- a/src/include/knl/knl_thread.h +++ b/src/include/knl/knl_thread.h @@ -3367,6 +3367,10 @@ typedef struct knl_t_ondemand_xlog_copy_context { uint32 openLogOff; } knl_t_ondemand_xlog_copy_context; +typedef struct knl_t_dms_auxiliary_context { + volatile sig_atomic_t shutdown_requested; +} knl_t_dms_auxiliary_context; + /* thread context. */ typedef struct knl_thrd_context { knl_thread_role role; @@ -3517,6 +3521,7 @@ typedef struct knl_thrd_context { knl_t_dms_context dms_cxt; knl_t_ondemand_xlog_copy_context ondemand_xlog_copy_cxt; knl_t_rc_context rc_cxt; + knl_t_dms_auxiliary_context dms_aux_cxt; } knl_thrd_context; #ifdef ENABLE_MOT diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h index a4ab2eb9d..dfee1d6db 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h @@ -563,6 +563,7 @@ typedef enum { TsCompactionAuxiliaryProcess, XlogCopyBackendProcess, BarrierPreParseBackendProcess, + DmsAuxiliaryProcess, NUM_SINGLE_AUX_PROC, /* Sentry for auxiliary type with single thread. */ /* diff --git a/src/test/ss/ha_test.sh b/src/test/ss/ha_test.sh index 0d799fb73..6e03d002a 100644 --- a/src/test/ss/ha_test.sh +++ b/src/test/ss/ha_test.sh @@ -131,7 +131,7 @@ function db_start_wait if [ ${temp} -eq 0 ]; then print "wait node $1 start ${wait_time}s" fi - if [ ${wait_time} -gt 600 ]; then + if [ ${wait_time} -gt 300 ]; then print "wait node $1 start ${wait_time}s, db start failed" exit 1 fi From 07ae9c52be2473ddc81f783d02d01c80eb16fa42 Mon Sep 17 00:00:00 2001 From: shenzheng4 Date: Thu, 3 Aug 2023 11:15:26 +0800 Subject: [PATCH 116/304] make_dorado_cluster_xlog_one_instance --- src/bin/initdb/initdb.cpp | 15 +- src/bin/initdb/ss_initdb.cpp | 48 +++- src/bin/initdb/ss_initdb.h | 4 +- src/bin/pg_ctl/pg_build.cpp | 4 + src/common/backend/utils/init/miscinit.cpp | 4 + .../backend/utils/misc/guc/guc_storage.cpp | 48 +++- .../utils/misc/postgresql_single.conf.sample | 1 + src/common/port/tool_common.cpp | 3 + src/gausskernel/bootstrap/bootstrap.cpp | 1 + .../ddes/adapter/ss_dms_bufmgr.cpp | 3 +- .../ddes/adapter/ss_dms_callback.cpp | 16 +- .../ddes/adapter/ss_dms_recovery.cpp | 5 +- .../ddes/adapter/ss_reform_common.cpp | 66 +++++ .../process/postmaster/checkpointer.cpp | 3 +- .../process/postmaster/postmaster.cpp | 81 ++++-- .../process/postmaster/startup.cpp | 3 +- .../process/threadpool/knl_instance.cpp | 4 + .../storage/access/transam/xlog.cpp | 240 +++++++++++++--- .../storage/dorado_operation/dorado_fd.cpp | 5 +- src/gausskernel/storage/ipc/procarray.cpp | 5 +- .../storage/replication/CMakeLists.txt | 1 + src/gausskernel/storage/replication/Makefile | 2 +- .../storage/replication/basebackup.cpp | 4 +- .../storage/replication/dataqueue.cpp | 3 +- .../storage/replication/datasyncrep.cpp | 3 +- .../storage/replication/libpqwalreceiver.cpp | 27 +- .../shared_storage_walreceiver.cpp | 57 ++-- .../replication/ss_cluster_replication.cpp | 264 ++++++++++++++++++ .../storage/replication/walreceiver.cpp | 29 +- .../storage/replication/walsender.cpp | 23 +- src/gausskernel/storage/smgr/segstore.cpp | 7 +- src/include/access/xlog.h | 6 +- src/include/access/xlog_basic.h | 2 + src/include/ddes/dms/ss_common_attr.h | 16 +- .../knl/knl_guc/knl_instance_attr_storage.h | 1 + src/include/knl/knl_instance.h | 2 + .../replication/shared_storage_walreceiver.h | 18 +- .../replication/ss_cluster_replication.h | 73 +++++ .../storage/dorado_operation/dorado_fd.h | 1 + src/include/tool_common.h | 1 + .../regress/output/recovery_2pc_tools.source | 1 + 41 files changed, 951 insertions(+), 149 deletions(-) create mode 100644 src/gausskernel/storage/replication/ss_cluster_replication.cpp create mode 100644 src/include/replication/ss_cluster_replication.h diff --git a/src/bin/initdb/initdb.cpp b/src/bin/initdb/initdb.cpp index f7812030f..7419849f7 100644 --- a/src/bin/initdb/initdb.cpp +++ b/src/bin/initdb/initdb.cpp @@ -173,6 +173,7 @@ static char* xlog_dir = ""; static bool security = false; static char* dbcompatibility = ""; static char* new_xlog_file_path = ""; +static bool enable_ss_dorado = false; #ifdef PGXC /* Name of the PGXC node initialized */ @@ -1595,6 +1596,12 @@ static void setup_config(void) conflines = ss_addnodeparmater(conflines); } + if (enable_ss_dorado) { + nRet = strcpy_s(repltok, sizeof(repltok), "enable_ss_dorado = on"); + securec_check_c(nRet, "\0", "\0"); + conflines = replace_token(conflines, "#enable_ss_dorado = off", repltok); + } + nRet = sprintf_s(path, sizeof(path), "%s/postgresql.conf", pg_data); securec_check_ss_c(nRet, "\0", "\0"); @@ -3976,6 +3983,7 @@ int main(int argc, char* argv[]) {"socketpath", required_argument, NULL, 16}, {"enable-dss", no_argument, NULL, 17}, {"dms_url", required_argument, NULL, 18}, + {"enable-ss-dorado", no_argument, NULL, 19}, {NULL, 0, NULL, 0}}; int c, i, ret; @@ -4291,6 +4299,10 @@ int main(int argc, char* argv[]) check_input_spec_char(optarg); ss_nodedatainfo = xstrdup(optarg); break; + case 19: + enable_ss_dorado = true; + printf(_("Enable ss dorado replication.\n")); + break; #endif default: /* getopt_long already emitted a complaint */ @@ -4915,7 +4927,8 @@ int main(int argc, char* argv[]) } if (enable_dss && ss_issharedstorage) { - ss_mkdirdir(ss_nodeid, pg_data, vgdata, vglog, ss_need_mkclusterdir); + ss_need_mkspecialdir = (enable_ss_dorado && !ss_check_specialdir(vgdata)); + ss_mkdirdir(ss_nodeid, pg_data, vgdata, vglog, ss_need_mkclusterdir, ss_need_mkspecialdir); } else { /* Create required subdirectories */ printf(_("creating subdirectories ... in ordinary occasion")); diff --git a/src/bin/initdb/ss_initdb.cpp b/src/bin/initdb/ss_initdb.cpp index 29ca9be4a..6285d9b78 100644 --- a/src/bin/initdb/ss_initdb.cpp +++ b/src/bin/initdb/ss_initdb.cpp @@ -77,16 +77,21 @@ static const char* ss_instanceowndirs[] = {"base", static const char* ss_xlogsubdirs[] = {"archive_status"}; +/* for ss dorado replication */ +static const char* ss_specialdirs[] = {"+pg_replication"}; + /* num of every directory type */ #define SS_CLUSTERDIRS_NUM ARRAY_NUM(ss_clusterdirs) #define SS_INSTANCEDIRS_NUM ARRAY_NUM(ss_instancedirs) #define SS_INSTANCEOENDIRS_NUM ARRAY_NUM(ss_instanceowndirs) #define SS_XLOGSUBIRS_NUM ARRAY_NUM(ss_xlogsubdirs) +#define SS_SPECIALDIRS_NUM ARRAY_NUM(ss_specialdirs) char *ss_nodedatainfo = NULL; int32 ss_nodeid = INVALID_INSTANCEID; bool ss_issharedstorage = false; bool ss_need_mkclusterdir = true; +bool ss_need_mkspecialdir = false; static const char *ss_progname = "ss_initdb"; /* @@ -224,6 +229,40 @@ int ss_check_existdir(const char *path, int node_id, const char **subdir) return 2; // subdir exists and complete } +bool ss_check_exist_specialdir(char *path) +{ + for (uint32_t i = 0; i < SS_SPECIALDIRS_NUM; ++i) { + if (strcmp(ss_specialdirs[i] + 1, path) == 0) { + return true; + } + } + return false; +} + +bool ss_check_specialdir(char *path) +{ + char *datadir = path; + DIR *pgdatadir = NULL; + struct dirent *file = NULL; + + if ((pgdatadir = opendir(datadir)) != NULL) { + while ((file = readdir(pgdatadir)) != NULL) { + if (strcmp(".", file->d_name) == 0 || strcmp("..", file->d_name) == 0) { + /* skip this and parent directory */ + continue; + } + + if (ss_check_exist_specialdir(file->d_name)) { + (void)closedir(pgdatadir); + return true; + } + } + (void)closedir(pgdatadir); + } + + return false; +} + int ss_check_shareddir(char *path, int node_id, bool *need_mkclusterdir) { int ret = 0; @@ -270,7 +309,7 @@ int ss_check_shareddir(char *path, int node_id, bool *need_mkclusterdir) } void ss_mkdirdir(int32 node_id, const char *pg_data, const char *vgdata_dir, const char *vglog_dir, - bool need_mkclusterdir) + bool need_mkclusterdir, bool need_specialdir) { /* Create required subdirectories */ printf(_("creating subdirectories ... in shared storage mode ... ")); @@ -286,6 +325,10 @@ void ss_mkdirdir(int32 node_id, const char *pg_data, const char *vgdata_dir, con if (need_mkclusterdir) { ss_createdir(ss_clusterdirs, SS_CLUSTERDIRS_NUM, INVALID_INSTANCEID, pg_data, vgdata_dir, vglog_dir); } + + if (need_specialdir) { + ss_createdir(ss_specialdirs, SS_SPECIALDIRS_NUM, INVALID_INSTANCEID, pg_data, vgdata_dir, vglog_dir); + } } void ss_makedirectory(char *path) @@ -334,7 +377,8 @@ void ss_createdir(const char **ss_dirs, int32 num, int32 node_id, const char *pg if (vglog_dir[0] != '\0' && (pg_strcasecmp(ss_dirs[i], "+pg_xlog") == 0 || pg_strcasecmp(ss_dirs[i], "+pg_doublewrite") == 0 || pg_strcasecmp(ss_dirs[i], "+pg_notify") == 0 || - pg_strcasecmp(ss_dirs[i], "+pg_snapshots") == 0)) { + pg_strcasecmp(ss_dirs[i], "+pg_snapshots") == 0 || + pg_strcasecmp(ss_dirs[i], "+pg_replication") == 0)) { is_xlog = true; } diff --git a/src/bin/initdb/ss_initdb.h b/src/bin/initdb/ss_initdb.h index ad299aab0..c29dc6a21 100644 --- a/src/bin/initdb/ss_initdb.h +++ b/src/bin/initdb/ss_initdb.h @@ -45,14 +45,16 @@ extern int32 ss_nodeid; extern bool ss_issharedstorage; extern bool ss_need_mkclusterdir; +extern bool ss_need_mkspecialdir; /* check dms url when gs_initdb */ extern bool ss_check_nodedatainfo(bool enable_dss); extern int ss_check_shareddir(char* path, int32 node_id, bool *ss_need_mkclusterdir); +extern bool ss_check_specialdir(char *path); extern void ss_createdir(const char** ss_dirs, int32 num, int32 node_id, const char* pg_data, const char* vgdata_dir, const char* vglog_dir); -extern void ss_mkdirdir(int32 node_id, const char* pg_data, const char* vgdata_dir, const char* vglog_dir, bool need_mkclusterdir); +extern void ss_mkdirdir(int32 node_id, const char* pg_data, const char* vgdata_dir, const char* vglog_dir, bool need_mkclusterdir, bool need_mkspecialdir); extern char** ss_addnodeparmater(char** conflines); diff --git a/src/bin/pg_ctl/pg_build.cpp b/src/bin/pg_ctl/pg_build.cpp index a84b1c2a3..451a2d017 100755 --- a/src/bin/pg_ctl/pg_build.cpp +++ b/src/bin/pg_ctl/pg_build.cpp @@ -1250,6 +1250,10 @@ bool SsIsSkipPath(const char* dirname, bool needskipall) return true; } + if (IsBeginWith(dirname, "pg_replication") > 0) { + return true; + } + /* skip pg_control file when dss enable, only copy pg_control of main standby, * we need to retain pg_control of other nodes, so pg_contol not be deleted directly. */ diff --git a/src/common/backend/utils/init/miscinit.cpp b/src/common/backend/utils/init/miscinit.cpp index aa5483b8d..b8e974640 100644 --- a/src/common/backend/utils/init/miscinit.cpp +++ b/src/common/backend/utils/init/miscinit.cpp @@ -2146,6 +2146,10 @@ void initDssPath(char *dssdir) dssdir); securec_check_ss(rc, "", ""); + rc = snprintf_s(g_instance.datadir_cxt.controlInfoPath, MAXPGPATH, MAXPGPATH - 1, "%s/pg_replication/pg_ss_ctl_info", + dssdir); + securec_check_ss(rc, "", ""); + ss_initdwsubdir(dssdir, g_instance.attr.attr_storage.dms_attr.instance_id); } diff --git a/src/common/backend/utils/misc/guc/guc_storage.cpp b/src/common/backend/utils/misc/guc/guc_storage.cpp index 7306eca39..e4144544d 100755 --- a/src/common/backend/utils/misc/guc/guc_storage.cpp +++ b/src/common/backend/utils/misc/guc/guc_storage.cpp @@ -216,6 +216,8 @@ static bool check_ss_dss_vg_name(char** newval, void** extra, GucSource source); static bool check_ss_dss_conn_path(char** newval, void** extra, GucSource source); static bool check_ss_enable_ssl(bool* newval, void** extra, GucSource source); static bool check_ss_enable_ondemand_recovery(bool* newval, void** extra, GucSource source); +static bool check_normal_cluster_replication_config_para(char** newval, void** extra, GucSource source); +static bool check_ss_cluster_replication_control_para(bool* newval, void** extra, GucSource source); #ifdef USE_ASSERT_CHECKING static void assign_ss_enable_verify_page(bool newval, void *extra); @@ -1129,6 +1131,18 @@ static void InitStorageConfigureNamesBool() NULL, NULL, NULL}, + {{"enable_ss_dorado", + PGC_POSTMASTER, + NODE_SINGLENODE, + WAL, + gettext_noop("Use to enabel dorado replication in share storage mode."), + NULL, + GUC_SUPERUSER_ONLY}, + &g_instance.attr.attr_storage.enable_ss_dorado, + false, + check_ss_cluster_replication_control_para, + NULL, + NULL}, #ifdef USE_ASSERT_CHECKING {{"enable_hashbucket", @@ -4424,7 +4438,7 @@ static void InitStorageConfigureNamesString() GUC_SUPERUSER_ONLY}, &g_instance.attr.attr_storage.xlog_file_path, NULL, - NULL, + check_normal_cluster_replication_config_para, NULL, NULL}, {{"hadr_super_user_record_path", @@ -5914,6 +5928,38 @@ static bool check_ss_rdma_work_config(char** newval, void** extra, GucSource sou return false; } +static bool check_normal_cluster_replication_config_para(char** newval, void** extra, GucSource source) +{ + if (newval == NULL || *newval == NULL || **newval == '\0') { + return true; + } + + if (g_instance.attr.attr_storage.enable_ss_dorado) { + ereport(ERROR, (errmsg("Do not allow both enable normal cluster replication " + "and ss cluster repliction with \"enable_ss_dorado\" = %d", \ + g_instance.attr.attr_storage.enable_ss_dorado))); + return false; + } + + return true; +} + +static bool check_ss_cluster_replication_control_para(bool* newval, void** extra, GucSource source) +{ + if (!(*newval)) { + return true; + } + + if (g_instance.attr.attr_storage.xlog_file_path != NULL) { + ereport(ERROR, (errmsg("Do not allow both enable ss cluster replication " + "and normal cluster repliction with \"xlog_file_path\" = %s", \ + g_instance.attr.attr_storage.xlog_file_path))); + return false; + } + + return true; +} + extern bool check_special_character(char c); static bool check_ss_ock_log_path(char **newval, void **extra, GucSource source) diff --git a/src/common/backend/utils/misc/postgresql_single.conf.sample b/src/common/backend/utils/misc/postgresql_single.conf.sample index 1194424f5..3b1eeb20a 100644 --- a/src/common/backend/utils/misc/postgresql_single.conf.sample +++ b/src/common/backend/utils/misc/postgresql_single.conf.sample @@ -851,3 +851,4 @@ job_queue_processes = 10 # Number of concurrent jobs, optional: [0..1000] #ss_parallel_thread_count = 16 #ss_enable_ondemand_recovery = off #ss_ondemand_recovery_mem_size = 4GB # min: 1GB, max: 100GB +#enable_ss_dorado = off diff --git a/src/common/port/tool_common.cpp b/src/common/port/tool_common.cpp index 81cb44d10..ffee72a16 100644 --- a/src/common/port/tool_common.cpp +++ b/src/common/port/tool_common.cpp @@ -149,6 +149,9 @@ static void initDSSDataPathStruct(datadir_t *dataDir) rc = snprintf_s(dataDir->controlBakPath, MAXPGPATH, MAXPGPATH - 1, "%s/pg_control.backup", dataDir->dss_data); securec_check_ss_c(rc, "", ""); + rc = snprintf_s(dataDir->controlInfoPath, MAXPGPATH, MAXPGPATH - 1, "%s/pg_replication/pg_ss_ctl_info", dataDir->dss_data); + securec_check_ss_c(rc, "", ""); + // DSS file directory (instance owner) rc = snprintf_s(dataDir->clogDir, MAXPGPATH, MAXPGPATH - 1, "%s/pg_clog%d", dataDir->dss_data, dataDir->instance_id); diff --git a/src/gausskernel/bootstrap/bootstrap.cpp b/src/gausskernel/bootstrap/bootstrap.cpp index 49eebd0f8..8f82f369c 100755 --- a/src/gausskernel/bootstrap/bootstrap.cpp +++ b/src/gausskernel/bootstrap/bootstrap.cpp @@ -308,6 +308,7 @@ void BootStrapProcessMain(int argc, char* argv[]) pfree(value); break; } + default: write_stderr("Try \"%s --help\" for more information.\n", progName); proc_exit(1); diff --git a/src/gausskernel/ddes/adapter/ss_dms_bufmgr.cpp b/src/gausskernel/ddes/adapter/ss_dms_bufmgr.cpp index ba644e984..917f63303 100644 --- a/src/gausskernel/ddes/adapter/ss_dms_bufmgr.cpp +++ b/src/gausskernel/ddes/adapter/ss_dms_bufmgr.cpp @@ -26,6 +26,7 @@ #include "storage/proc.h" #include "storage/buf/bufmgr.h" #include "storage/smgr/segment.h" +#include "replication/ss_cluster_replication.h" #include "utils/resowner.h" #include "ddes/dms/ss_dms_bufmgr.h" #include "ddes/dms/ss_reform_common.h" @@ -741,7 +742,7 @@ dms_session_e DMSGetProcType4RequestPage() * DMS_SESSION_RECOVER_HOT_STANDBY will be returned, it indicates that normal threads can access * page in recovery state. */ - if (SS_STANDBY_CLUSTER_MAIN_STANDBY && pmState == PM_HOT_STANDBY) { + if ((SS_STANDBY_CLUSTER_MAIN_STANDBY || IS_SS_REPLICATION_MAIN_STANBY_NODE) && pmState == PM_HOT_STANDBY) { return DMS_SESSION_RECOVER_HOT_STANDBY; } else { return DMS_SESSION_RECOVER; diff --git a/src/gausskernel/ddes/adapter/ss_dms_callback.cpp b/src/gausskernel/ddes/adapter/ss_dms_callback.cpp index 6ebada131..58af24139 100644 --- a/src/gausskernel/ddes/adapter/ss_dms_callback.cpp +++ b/src/gausskernel/ddes/adapter/ss_dms_callback.cpp @@ -41,6 +41,7 @@ #include "storage/sinvaladt.h" #include "replication/walsender_private.h" #include "replication/walreceiver.h" +#include "replication/ss_cluster_replication.h" #include "ddes/dms/ss_switchover.h" #include "ddes/dms/ss_reform_common.h" #include "ddes/dms/ss_dms_bufmgr.h" @@ -57,7 +58,7 @@ void SSWakeupRecovery(void) /* need make sure pagewriter started first */ bool need_recovery = true; - if (DORADO_STANDBY_CLUSTER) { + if (DORADO_STANDBY_CLUSTER || SS_REPLICATION_STANDBY_CLUSTER) { g_instance.dms_cxt.SSRecoveryInfo.recovery_pause_flag = false; return; } @@ -200,7 +201,8 @@ static int CBGetTxnCSN(void *db_handle, dms_opengauss_xid_csn_t *csn_req, dms_op static int CBGetSnapshotData(void *db_handle, dms_opengauss_txn_snapshot_t *txn_snapshot, uint8 inst_id) { /* SS_STANDBY_CLUSTER_NORMAL_MAIN_STANDBY always is in recovery progress, but it can acquire snapshot*/ - if (RecoveryInProgress() && !SS_STANDBY_CLUSTER_NORMAL_MAIN_STANDBY) { + if (RecoveryInProgress() && + !(SS_STANDBY_CLUSTER_NORMAL_MAIN_STANDBY || (SS_NORMAL_PRIMARY && IS_SS_REPLICATION_MAIN_STANBY_NODE))) { return DMS_ERROR; } @@ -419,7 +421,7 @@ static void CBSwitchoverResult(void *db_handle, int result) } else { /* abort and restore state */ g_instance.dms_cxt.SSClusterState = NODESTATE_NORMAL; - if (DORADO_STANDBY_CLUSTER) { + if (DORADO_STANDBY_CLUSTER || SS_REPLICATION_STANDBY_CLUSTER) { g_instance.dms_cxt.SSReformInfo.in_reform = false; } ereport(WARNING, (errmodule(MOD_DMS), errmsg("[SS switchover] Switchover failed, errno: %d.", result))); @@ -1577,7 +1579,7 @@ static void CBReformSetDmsRole(void *db_handle, unsigned int reformer_id) if (new_dms_role == DMS_ROLE_REFORMER) { ereport(LOG, (errmodule(MOD_DMS), errmsg("[SS switchover]begin to set currrent DSS as primary"))); /* standby of standby cluster need to set mode to STANDBY_MODE in dual cluster*/ - if (DORADO_STANDBY_CLUSTER) { + if (DORADO_STANDBY_CLUSTER || SS_REPLICATION_STANDBY_CLUSTER) { t_thrd.postmaster_cxt.HaShmData->current_mode = STANDBY_MODE; } while (dss_set_server_status_wrapper() != GS_SUCCESS) { @@ -1753,7 +1755,7 @@ static void CBReformStartNotify(void *db_handle, dms_role_t role, unsigned char /* After reform done, standby of standby cluster need to set mode to STANDBY_MODE in dual cluster. */ if (SS_REFORM_REFORMER && (g_instance.attr.attr_common.cluster_run_mode == RUN_MODE_STANDBY) && - (g_instance.attr.attr_storage.xlog_file_path != 0)) { + (g_instance.attr.attr_storage.xlog_file_path != NULL || SS_CLUSTER_DORADO_REPLICATION)) { t_thrd.postmaster_cxt.HaShmData->current_mode = STANDBY_MODE; } } @@ -1770,7 +1772,7 @@ static int CBReformDoneNotify(void *db_handle) /* After reform done, primary of master cluster need to set mode to PRIMARY_MODE in dual cluster. */ if (SS_REFORM_REFORMER && (g_instance.attr.attr_common.cluster_run_mode == RUN_MODE_PRIMARY) && - (g_instance.attr.attr_storage.xlog_file_path != 0)) { + (g_instance.attr.attr_storage.xlog_file_path != NULL || SS_CLUSTER_DORADO_REPLICATION)) { t_thrd.postmaster_cxt.HaShmData->current_mode = PRIMARY_MODE; } @@ -1853,7 +1855,7 @@ void DmsCallbackThreadShmemInit(unsigned char need_startup, char **reg_data) { /* in dorado mode, we need to wait sharestorageinit finished */ while (!g_instance.dms_cxt.SSRecoveryInfo.dorado_sharestorage_inited && - g_instance.attr.attr_storage.xlog_file_path != 0) { + (g_instance.attr.attr_storage.xlog_file_path != NULL || SS_CLUSTER_DORADO_REPLICATION)) { pg_usleep(REFORM_WAIT_TIME); } IsUnderPostmaster = true; diff --git a/src/gausskernel/ddes/adapter/ss_dms_recovery.cpp b/src/gausskernel/ddes/adapter/ss_dms_recovery.cpp index de58c272b..48dd3646b 100644 --- a/src/gausskernel/ddes/adapter/ss_dms_recovery.cpp +++ b/src/gausskernel/ddes/adapter/ss_dms_recovery.cpp @@ -33,6 +33,7 @@ #include "storage/smgr/segment.h" #include "postmaster/postmaster.h" #include "storage/file/fio_device.h" +#include "replication/ss_cluster_replication.h" #include "ddes/dms/ss_dms_bufmgr.h" #include "ddes/dms/ss_dms_recovery.h" #include "ddes/dms/ss_reform_common.h" @@ -125,7 +126,7 @@ bool SSRecoveryNodes() * recovery phase could be regarded successful in hot_standby thus set pmState = PM_HOT_STANDBY, which * indicate database systerm is ready to accept read only connections. */ - if (SS_STANDBY_CLUSTER_MAIN_STANDBY && pmState == PM_HOT_STANDBY) { + if ((SS_STANDBY_CLUSTER_MAIN_STANDBY || IS_SS_REPLICATION_MAIN_STANBY_NODE) && pmState == PM_HOT_STANDBY) { result = true; break; } @@ -146,7 +147,7 @@ bool SSRecoveryApplyDelay() return false; } - if (DORADO_STANDBY_CLUSTER) { + if (DORADO_STANDBY_CLUSTER || SS_REPLICATION_STANDBY_CLUSTER) { return true; } diff --git a/src/gausskernel/ddes/adapter/ss_reform_common.cpp b/src/gausskernel/ddes/adapter/ss_reform_common.cpp index d0dee7cf1..cca457cb5 100644 --- a/src/gausskernel/ddes/adapter/ss_reform_common.cpp +++ b/src/gausskernel/ddes/adapter/ss_reform_common.cpp @@ -36,6 +36,33 @@ #include "storage/file/fio_device.h" #include "storage/smgr/segment_internal.h" #include "replication/walreceiver.h" +#include "replication/ss_cluster_replication.h" + +/* + * Add xlog reader private structure for page read. + */ +typedef struct XLogPageReadPrivate { + int emode; + bool fetching_ckpt; /* are we fetching a checkpoint record? */ + bool randAccess; +} XLogPageReadPrivate; + +std::vector SSGetAllStableNodeId() +{ + std::vector posList; + int pos = 0; + uint64 stableInstId = g_instance.dms_cxt.SSReformerControl.list_stable; + while (stableInstId) { + uint64 res = stableInstId & 0x01; + if (res) { + posList.emplace_back(pos); + } + pos++; + stableInstId = stableInstId >> 1; + } + + return posList; +} int SSXLogFileReadAnyTLI(XLogSegNo segno, int emode, uint32 sources, char* xlog_path) { @@ -56,6 +83,7 @@ int SSXLogFileReadAnyTLI(XLogSegNo segno, int emode, uint32 sources, char* xlog_ t_thrd.xlog_cxt.restoredFromArchive = false; fd = BasicOpenFile(path, O_RDONLY | PG_BINARY, 0); +retry: if (fd >= 0) { /* Success! */ t_thrd.xlog_cxt.curFileTLI = tli; @@ -71,6 +99,44 @@ int SSXLogFileReadAnyTLI(XLogSegNo segno, int emode, uint32 sources, char* xlog_ return fd; } + + /* + * When SS_CLUSTER_DORADO_REPLICATION enabled, current xlog dictionary may be not the correct dictionary, + * because all xlog dictionaries are in the same LUN, we need loop over other dictionaries. + * Do we need source == XLOG_FROM_STREAM? + */ + if (SS_CLUSTER_DORADO_REPLICATION) { + std::vector nodeList = SSGetAllStableNodeId(); // stable node list, + Assert(!nodeList.empty()); + char xlogPath[MAXPGPATH]; + char *dssdir = g_instance.attr.attr_storage.dss_attr.ss_dss_vg_name; + for (auto elem : nodeList) { + if (elem == g_instance.dms_cxt.SSReformerControl.recoveryInstId) { + continue; + } + + errorno = memset_s(xlogPath, sizeof(xlogPath), 0, sizeof(xlogPath)); + securec_check_ss(errorno, "", ""); + /* try to read from other xlog dictionary */ + errorno = snprintf_s(xlogPath, MAXPGPATH, MAXPGPATH - 1, "%s/pg_xlog%d", dssdir, elem); + securec_check_ss(errorno, "", ""); + + errorno = memset_s(path, sizeof(path), 0, sizeof(path)); + securec_check_ss(errorno, "", ""); + + errorno = snprintf_s(path, MAXPGPATH, MAXPGPATH - 1, "%s/%08X%08X%08X", xlogPath, tli, + (uint32)((segno) / XLogSegmentsPerXLogId), (uint32)((segno) % XLogSegmentsPerXLogId)); + securec_check_ss(errorno, "", ""); + + fd = BasicOpenFile(path, O_RDONLY | PG_BINARY, 0); + if (fd < 0) { + continue; + } + ereport(LOG, (errmsg("find xlog file in path : \"%s\"", path))); + goto retry; + } + } + if (!FILE_POSSIBLY_DELETED(errno)) { ereport(PANIC, (errcode_for_file_access(), errmsg("could not open file \"%s\" (log segment %s): %m", path, XLogFileNameP(t_thrd.xlog_cxt.ThisTimeLineID, segno)))); diff --git a/src/gausskernel/process/postmaster/checkpointer.cpp b/src/gausskernel/process/postmaster/checkpointer.cpp index a5084828c..527f81c23 100755 --- a/src/gausskernel/process/postmaster/checkpointer.cpp +++ b/src/gausskernel/process/postmaster/checkpointer.cpp @@ -47,6 +47,7 @@ #include "postmaster/bgwriter.h" #include "postmaster/pagewriter.h" #include "replication/syncrep.h" +#include "replication/ss_cluster_replication.h" #include "storage/buf/bufmgr.h" #include "storage/ipc.h" #include "storage/lock/lwlock.h" @@ -542,7 +543,7 @@ void CheckpointerMain(void) } else { CheckPointBuffers(flags, true); } - } else if (!do_restartpoint && !DORADO_STANDBY_CLUSTER_MAINSTANDBY_NODE) { + } else if (!do_restartpoint && !(DORADO_STANDBY_CLUSTER_MAINSTANDBY_NODE || IS_SS_REPLICATION_MAIN_STANBY_NODE)) { CreateCheckPoint(flags); ckpt_performed = true; if (!bgwriter_first_startup && CheckFpwBeforeFirstCkpt()) { diff --git a/src/gausskernel/process/postmaster/postmaster.cpp b/src/gausskernel/process/postmaster/postmaster.cpp index 1c60bfb6f..dc04bb74e 100644 --- a/src/gausskernel/process/postmaster/postmaster.cpp +++ b/src/gausskernel/process/postmaster/postmaster.cpp @@ -149,6 +149,7 @@ #include "replication/dcf_replication.h" #include "replication/logicallauncher.h" #include "replication/logicalworker.h" +#include "replication/ss_cluster_replication.h" #include "postmaster/bgwriter.h" #include "postmaster/cbmwriter.h" #include "postmaster/startup.h" @@ -3690,6 +3691,32 @@ void ArchObsThreadManage() SpinLockRelease(&g_instance.archive_obs_cxt.mutex); } +static bool IsNeedStartXlogCopyer() +{ + /* Only in normal cluster replication and ss cluster replication, we need start xlogcopyer thread. */ + if (SS_CLUSTER_DORADO_REPLICATION || !IS_SHARED_STORAGE_MODE) { + return false; + } + + if (g_instance.pid_cxt.sharedStorageXlogCopyThreadPID != 0 || dummyStandbyMode) { + return false; + } + + /* Non SS mode */ + if (!ENABLE_DMS && (pmState == PM_RUN || t_thrd.xlog_cxt.is_hadr_main_standby)) { + return true; + } + + /* SS mode, Primary cluster primary node need start xlogcopyer thread. */ + if (ENABLE_DMS && pmState == PM_RUN && + t_thrd.postmaster_cxt.HaShmData->current_mode == PRIMARY_MODE && + !SS_PERFORMING_SWITCHOVER) { + return true; + } + + return false; +} + /* * Main idle loop of postmaster */ @@ -3740,7 +3767,7 @@ static int ServerLoop(void) fd_set rmask; int selres; - if (t_thrd.postmaster_cxt.HaShmData->current_mode != NORMAL_MODE || IS_SHARED_STORAGE_MODE) { + if (t_thrd.postmaster_cxt.HaShmData->current_mode != NORMAL_MODE || IS_SHARED_STORAGE_MODE || SS_CLUSTER_DORADO_REPLICATION) { check_and_reset_ha_listen_port(); #ifdef HAVE_POLL @@ -4008,11 +4035,7 @@ static int ServerLoop(void) if (g_instance.pid_cxt.ReaperBackendPID == 0) g_instance.pid_cxt.ReaperBackendPID = initialize_util_thread(REAPER); - if (((!ENABLE_DMS && (pmState == PM_RUN || t_thrd.xlog_cxt.is_hadr_main_standby)) || - (ENABLE_DMS && pmState == PM_RUN && t_thrd.postmaster_cxt.HaShmData->current_mode == PRIMARY_MODE && - !SS_PERFORMING_SWITCHOVER)) && - g_instance.pid_cxt.sharedStorageXlogCopyThreadPID == 0 && !dummyStandbyMode && - g_instance.attr.attr_storage.xlog_file_path != NULL) { + if (IsNeedStartXlogCopyer()) { g_instance.pid_cxt.sharedStorageXlogCopyThreadPID = initialize_util_thread(SHARE_STORAGE_XLOG_COPYER); } @@ -6605,7 +6628,7 @@ static void ProcessDemoteRequest(void) signal_child(g_instance.pid_cxt.StatementPID, SIGTERM); } - if (g_instance.pid_cxt.StartupPID != 0 && DORADO_STANDBY_CLUSTER) { + if (g_instance.pid_cxt.StartupPID != 0 && (DORADO_STANDBY_CLUSTER || SS_REPLICATION_STANDBY_CLUSTER)) { signal_child(g_instance.pid_cxt.StartupPID, SIGTERM); } @@ -6614,7 +6637,7 @@ static void ProcessDemoteRequest(void) signal_child(g_instance.pid_cxt.WalWriterPID, SIGTERM); StopAliveBuildSender(); - if (g_instance.pid_cxt.WalReceiverPID != 0 && DORADO_STANDBY_CLUSTER) { + if (g_instance.pid_cxt.WalReceiverPID != 0 && SS_REPLICATION_STANDBY_CLUSTER) { signal_child(g_instance.pid_cxt.WalReceiverPID, SIGTERM); } @@ -10050,12 +10073,17 @@ static void sigusr1_handler(SIGNAL_ARGS) StartPgjobWorker(); } - /* if xlog_file_path is not equel to zero and dms is enabled, main standby need to initialize walreceiver - * and walrecwrite. Other modes don't need when dms is enabled. */ + /* + * 1. if xlog_file_path is not equel to zero and dms is enabled, main standby need to initialize walreceiver + * and walrecwrite. Other modes don't need when dms is enabled. + * 2. if ss dorado replication enabled, don't need walrecwriter thread + */ if (CheckPostmasterSignal(PMSIGNAL_START_WALRECEIVER) && g_instance.pid_cxt.WalReceiverPID == 0 && (pmState == PM_STARTUP || pmState == PM_RECOVERY || pmState == PM_HOT_STANDBY || pmState == PM_WAIT_READONLY) && - g_instance.status == NoShutdown && (!ENABLE_DMS || SS_STANDBY_CLUSTER_MAIN_STANDBY)) { - if (g_instance.pid_cxt.WalRcvWriterPID == 0) { + g_instance.status == NoShutdown && + (!ENABLE_DMS || SS_STANDBY_CLUSTER_MAIN_STANDBY || SS_CLUSTER_DORADO_REPLICATION)) { + /* when SS_CLUSTER_DORADO_REPLICATION enabled, don't start walrecwrite */ + if (g_instance.pid_cxt.WalRcvWriterPID == 0 && !SS_CLUSTER_DORADO_REPLICATION) { g_instance.pid_cxt.WalRcvWriterPID = initialize_util_thread(WALRECWRITE); SetWalRcvWriterPID(g_instance.pid_cxt.WalRcvWriterPID); } @@ -10065,7 +10093,8 @@ static void sigusr1_handler(SIGNAL_ARGS) } if (CheckPostmasterSignal(PMSIGNAL_START_DATARECEIVER) && g_instance.pid_cxt.DataReceiverPID == 0 && - (pmState == PM_STARTUP || pmState == PM_RECOVERY || pmState == PM_HOT_STANDBY || pmState == PM_WAIT_READONLY) && + (pmState == PM_STARTUP || pmState == PM_RECOVERY || pmState == PM_HOT_STANDBY || + pmState == PM_WAIT_READONLY) && g_instance.status == NoShutdown) { if (g_instance.pid_cxt.DataRcvWriterPID == 0) { g_instance.pid_cxt.DataRcvWriterPID = initialize_util_thread(DATARECWRITER); @@ -10098,7 +10127,8 @@ static void sigusr1_handler(SIGNAL_ARGS) } if ((mode = CheckSwitchoverSignal()) != 0 && WalRcvIsOnline() && DataRcvIsOnline() && - (pmState == PM_STARTUP || pmState == PM_RECOVERY || pmState == PM_HOT_STANDBY || pmState == PM_WAIT_READONLY)) { + (pmState == PM_STARTUP || pmState == PM_RECOVERY || pmState == PM_HOT_STANDBY || + pmState == PM_WAIT_READONLY)) { if (!IS_SHARED_STORAGE_STANDBY_CLUSTER_STANDBY_MODE) { ereport(LOG, (errmsg("to do switchover"))); /* Label the standby to do switchover */ @@ -10139,17 +10169,16 @@ static void sigusr1_handler(SIGNAL_ARGS) if (ENABLE_THREAD_POOL) { g_threadPoolControler->CloseAllSessions(); /* - * before pmState set to wait backends, - * threadpool cannot launch new thread by scheduler during demote. - */ + * before pmState set to wait backends, + * threadpool cannot launch new thread by scheduler during demote. + */ g_threadPoolControler->ShutDownScheduler(true, true); g_threadPoolControler->ShutDownThreads(true); } /* shut down all backends and autovac workers */ (void)SignalSomeChildren(SIGTERM, BACKEND_TYPE_NORMAL | BACKEND_TYPE_AUTOVAC); - if (g_instance.pid_cxt.PgStatPID != 0 && - g_instance.attr.attr_common.cluster_run_mode == RUN_MODE_STANDBY) { + if (g_instance.pid_cxt.PgStatPID != 0 && g_instance.attr.attr_common.cluster_run_mode == RUN_MODE_STANDBY) { signal_child(g_instance.pid_cxt.PgStatPID, SIGQUIT); } @@ -10185,23 +10214,23 @@ static void sigusr1_handler(SIGNAL_ARGS) signal_child(g_instance.pid_cxt.WLMCollectPID, SIGTERM); } - if (g_instance.pid_cxt.UndoLauncherPID != 0 && DORADO_STANDBY_CLUSTER) { + if (g_instance.pid_cxt.UndoLauncherPID != 0 && (DORADO_STANDBY_CLUSTER ||SS_REPLICATION_STANDBY_CLUSTER)) { signal_child(g_instance.pid_cxt.UndoLauncherPID, SIGTERM); } #ifndef ENABLE_MULTIPLE_NODES - if (g_instance.pid_cxt.ApplyLauncerPID != 0 && DORADO_STANDBY_CLUSTER) { + if (g_instance.pid_cxt.ApplyLauncerPID != 0 && (DORADO_STANDBY_CLUSTER ||SS_REPLICATION_STANDBY_CLUSTER)) { signal_child(g_instance.pid_cxt.ApplyLauncerPID, SIGTERM); } #endif - if (g_instance.pid_cxt.GlobalStatsPID != 0 && DORADO_STANDBY_CLUSTER) { + if (g_instance.pid_cxt.GlobalStatsPID != 0 && (DORADO_STANDBY_CLUSTER ||SS_REPLICATION_STANDBY_CLUSTER)) { signal_child(g_instance.pid_cxt.GlobalStatsPID, SIGTERM); } - if (g_instance.pid_cxt.UndoRecyclerPID != 0 && DORADO_STANDBY_CLUSTER) { + if (g_instance.pid_cxt.UndoRecyclerPID != 0 && (DORADO_STANDBY_CLUSTER ||SS_REPLICATION_STANDBY_CLUSTER)) { signal_child(g_instance.pid_cxt.UndoRecyclerPID, SIGTERM); } - if (g_instance.pid_cxt.FaultMonitorPID != 0 && DORADO_STANDBY_CLUSTER) { + if (g_instance.pid_cxt.FaultMonitorPID != 0 && (DORADO_STANDBY_CLUSTER ||SS_REPLICATION_STANDBY_CLUSTER)) { signal_child(g_instance.pid_cxt.FaultMonitorPID, SIGTERM); } @@ -12639,7 +12668,7 @@ const char* wal_get_db_state_string(DbState db_state) static ServerMode get_cur_mode(void) { if (ENABLE_DMS) { - if (DORADO_STANDBY_CLUSTER) { + if (DORADO_STANDBY_CLUSTER || SS_REPLICATION_STANDBY_CLUSTER) { return STANDBY_MODE; } /* except for main standby in standby cluster, current mode of instance is determined by SS_OFFICIAL_PRIMARY*/ @@ -14911,7 +14940,7 @@ void InitShmemForDmsCallBack() const char *GetSSServerMode(ServerMode mode) { - if (g_instance.attr.attr_storage.xlog_file_path != 0) { + if (IS_SHARED_STORAGE_MODE || SS_CLUSTER_DORADO_REPLICATION) { if (SS_OFFICIAL_PRIMARY && (mode == PRIMARY_MODE || mode == NORMAL_MODE)) { return "Primary"; } diff --git a/src/gausskernel/process/postmaster/startup.cpp b/src/gausskernel/process/postmaster/startup.cpp index 86a234741..2a47f3027 100755 --- a/src/gausskernel/process/postmaster/startup.cpp +++ b/src/gausskernel/process/postmaster/startup.cpp @@ -29,6 +29,7 @@ #include "postmaster/startup.h" #include "postmaster/postmaster.h" #include "replication/shared_storage_walreceiver.h" +#include "replication/ss_cluster_replication.h" #include "storage/ipc.h" #include "storage/latch.h" #include "storage/pmsignal.h" @@ -284,7 +285,7 @@ static void StartupReleaseAllLocks(int code, Datum arg) void DeleteDisConnFileInClusterStandby() { - if (!IS_SHARED_STORAGE_MODE) { + if (!(IS_SHARED_STORAGE_MODE || SS_CLUSTER_DORADO_REPLICATION)) { return; } diff --git a/src/gausskernel/process/threadpool/knl_instance.cpp b/src/gausskernel/process/threadpool/knl_instance.cpp index 09b54dd36..231f78a2b 100755 --- a/src/gausskernel/process/threadpool/knl_instance.cpp +++ b/src/gausskernel/process/threadpool/knl_instance.cpp @@ -469,6 +469,7 @@ static void knl_g_xlog_init(knl_g_xlog_context *xlog_cxt) securec_check(rc, "\0", "\0"); pthread_mutex_init(&xlog_cxt->remain_segs_lock, NULL); xlog_cxt->shareStorageLockFd = -1; + xlog_cxt->ssReplicationXLogCtl = NULL; } static void KnlGUndoInit(knl_g_undo_context *undoCxt) @@ -849,6 +850,9 @@ static void knl_g_datadir_init(knl_g_datadir_context* datadir_init) errorno = strcpy_s(datadir_init->controlBakPath, MAXPGPATH, "global/pg_control.backup"); securec_check_c(errorno, "\0", "\0"); + errorno = strcpy_s(datadir_init->controlInfoPath, MAXPGPATH, "pg_replication/pg_ss_ctl_info"); + securec_check_c(errorno, "\0", "\0"); + knl_g_dwsubdir_init(&datadir_init->dw_subdir_cxt); } diff --git a/src/gausskernel/storage/access/transam/xlog.cpp b/src/gausskernel/storage/access/transam/xlog.cpp index 547e4850d..c2a78ee19 100755 --- a/src/gausskernel/storage/access/transam/xlog.cpp +++ b/src/gausskernel/storage/access/transam/xlog.cpp @@ -87,6 +87,7 @@ #include "replication/reorderbuffer.h" #include "replication/replicainternal.h" #include "replication/shared_storage_walreceiver.h" +#include "replication/ss_cluster_replication.h" #include "replication/slot.h" #include "replication/snapbuild.h" #include "replication/syncrep.h" @@ -264,6 +265,7 @@ static void remove_xlogtemp_files(void); static bool validate_parse_delay_ddl_file(DelayDDLRange *delayRange); static bool write_delay_ddl_file(const DelayDDLRange &delayRange, bool onErrDelete); extern void CalculateLocalLatestSnapshot(bool forceCalc); +extern std::vector SSGetAllStableNodeId(); /* * Calculate the amount of space left on the page after 'endptr'. Beware @@ -2791,6 +2793,7 @@ static void XLogWrite(const XLogwrtRqst &WriteRqst, bool flexible) /* signal that we need to wakeup walsenders later */ WalSndWakeupRequest(); t_thrd.xlog_cxt.LogwrtResult->Flush = t_thrd.xlog_cxt.LogwrtResult->Write; /* end of page */ + UpdateSSDoradoCtlInfoAndSync(); AddShareStorageXLopCopyBackendWakeupRequest(); if (XLogArchivingActive()) { XLogArchiveNotifySeg(t_thrd.xlog_cxt.openLogSegNo); @@ -2863,6 +2866,7 @@ static void XLogWrite(const XLogwrtRqst &WriteRqst, bool flexible) /* signal that we need to wakeup walsenders later */ WalSndWakeupRequest(); t_thrd.xlog_cxt.LogwrtResult->Flush = t_thrd.xlog_cxt.LogwrtResult->Write; + UpdateSSDoradoCtlInfoAndSync(); AddShareStorageXLopCopyBackendWakeupRequest(); } @@ -4286,6 +4290,7 @@ static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, int source, } fd = BasicOpenFile(path, O_RDONLY | PG_BINARY, 0); +retry: if (fd >= 0) { /* Success! */ t_thrd.xlog_cxt.curFileTLI = tli; @@ -4305,6 +4310,39 @@ static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, int source, return fd; } + + /* + * When SS_CLUSTER_DORADO_REPLICATION enabled, current xlog dictionary may be not the correct dictionary, + * because all xlog dictionaries are in the same LUN, we need loop over other dictionaries. + * Do we need source == XLOG_FROM_STREAM? + */ + if (SS_CLUSTER_DORADO_REPLICATION && source == XLOG_FROM_STREAM) { + std::vector nodeList = SSGetAllStableNodeId(); // stable node list, + Assert(!nodeList.empty()); + char xlogPath[MAXPGPATH]; + char *dssdir = g_instance.attr.attr_storage.dss_attr.ss_dss_vg_name; + for (auto elem : nodeList) { + if (elem == g_instance.dms_cxt.SSReformerControl.recoveryInstId) { + continue; + } + + /* try to read from other xlog dictionary */ + errorno = snprintf_s(xlogPath, MAXPGPATH, MAXPGPATH - 1, "%s/pg_xlog%d", dssdir, elem); + securec_check_ss(errorno, "", ""); + + errorno = snprintf_s(path, MAXPGPATH, MAXPGPATH - 1, "%s/%08X%08X%08X", xlogPath, tli, + (uint32)((segno) / XLogSegmentsPerXLogId), (uint32)((segno) % XLogSegmentsPerXLogId)); + securec_check_ss(errorno, "", ""); + + fd = BasicOpenFile(path, O_RDONLY | PG_BINARY, 0); + if (fd < 0) { + continue; + } + ereport(LOG, (errmsg("find xlog file in path : \"%s\"", path))); + goto retry; + } + } + if (!FILE_POSSIBLY_DELETED(errno) || !notfoundOk) { /* unexpected failure? */ ereport(PANIC, (errcode_for_file_access(), errmsg("could not open file \"%s\" (log segment %s): %m", path, XLogFileNameP(t_thrd.xlog_cxt.ThisTimeLineID, segno)))); @@ -6537,6 +6575,35 @@ static uint64 GetMACAddr(void) return macAddr; } +void UpdateCtlInfoToFile(ShareStorageXLogCtl *ctlInfo) +{ + int fd = BasicOpenFile(SS_DORADO_CTRL_FILE, O_RDWR | PG_BINARY | O_DIRECT, S_IRUSR | S_IWUSR); + if (fd < 0) { + ereport(FATAL, (errcode_for_file_access(), errmsg("could not open ss ctl into file \"%s\": %m", SS_DORADO_CTRL_FILE))); + } + + Assert(fd > 0); + + errno = EOK; + if (pwrite(fd, ctlInfo, SS_DORADO_CTL_INFO_SIZE, 0) != SS_DORADO_CTL_INFO_SIZE) { + if (errno == 0) { + errno = ENOSPC; + } + ereport(PANIC, (errcode_for_file_access(), errmsg("could not write to ss ctl info file: %m"))); + } + + if (pg_fsync(fd) != 0) { + int save_errno = errno; + close(fd); + errno = save_errno; + ereport(PANIC, (errcode_for_file_access(), errmsg("could not fsync ss ctl info file: %m"))); + } + + if (close(fd)) { + ereport(ERROR, (errcode_for_file_access(), errmsg("could not close file \"%s\": %m", SS_DORADO_CTRL_FILE))); + } +} + /* * This func must be called ONCE on system install. It creates pg_control * and the initial XLOG segment. @@ -6738,6 +6805,18 @@ void BootStrapXLOG(void) errmsg("update dorado ctl info: len:%u, version:%u, start:%lu, end:%lu", ctlInfo->length, ctlInfo->version, ctlInfo->insertHead, ctlInfo->insertTail))); } + + if (SS_CLUSTER_DORADO_REPLICATION) { + ShareStorageXLogCtl *ctlInfo = g_instance.xlog_cxt.ssReplicationXLogCtl; + InitSSDoradoCtlInfo(ctlInfo, sysidentifier); + ctlInfo->insertHead = MAXALIGN((recptr - (char *)page)); + ctlInfo->crc = CalShareStorageCtlInfoCrc(ctlInfo); + WriteSSDoradoCtlInfoFile(); + ereport(LOG, (errcode_for_file_access(), + errmsg("update ss dorado ctl info: len:%u, version:%u, start:%lu", ctlInfo->length, + ctlInfo->version, ctlInfo->insertHead))); + } + #ifndef ENABLE_MULTIPLE_NODES if (g_instance.attr.attr_storage.dcf_attr.enable_dcf) { ret = memset_s(t_thrd.shemem_ptr_cxt.dcfData, sizeof(DCFData), 0, sizeof(DCFData)); @@ -10145,6 +10224,11 @@ void StartupXLOG(void) GetWritePermissionSharedStorage(); CheckShareStorageCtlInfo(EndOfLog); } + + if (IS_SS_REPLICATION_PRIMARY_NODE) { + CheckSSDoradoCtlInfo(EndOfLog); + } + /* * Complain if we did not roll forward far enough to render the backup @@ -10391,6 +10475,7 @@ void StartupXLOG(void) /* Shut down readFile facility, free space. */ ShutdownReadFileFacility(); + /* When ss dorado enabled, and standby promoting, we don't need to copy */ if (SS_STANDBY_FAILOVER && SS_PRIMARY_CLUSTER_STANDBY) { ereport(LOG, (errmodule(MOD_DMS), errmsg("[SS failover] standby promoting: copy endofxlog pageptr from primary to standby"))); @@ -10576,7 +10661,8 @@ void StartupXLOG(void) } #endif - if (ENABLE_DMS && ENABLE_REFORM && !SS_PRIMARY_DEMOTED && !DORADO_STANDBY_CLUSTER) { + if (ENABLE_DMS && ENABLE_REFORM && !SS_PRIMARY_DEMOTED && + !(DORADO_STANDBY_CLUSTER || SS_REPLICATION_STANDBY_CLUSTER)) { StartupWaitReform(); } } @@ -11296,7 +11382,7 @@ void ShutdownXLOG(int code, Datum arg) { if (SS_PRIMARY_DEMOTING) { ereport(LOG, (errmsg("[SS switchover] primary demote: doing shutdown checkpoint"))); - if (DORADO_STANDBY_CLUSTER) { + if (DORADO_STANDBY_CLUSTER || SS_REPLICATION_STANDBY_CLUSTER) { CreateRestartPoint(CHECKPOINT_IS_SHUTDOWN | CHECKPOINT_IMMEDIATE); } else { CreateCheckPoint(CHECKPOINT_IS_SHUTDOWN | CHECKPOINT_IMMEDIATE); @@ -11329,7 +11415,7 @@ void ShutdownXLOG(int code, Datum arg) } if (g_instance.wal_cxt.upgradeSwitchMode != ExtremelyFast) { - if (DORADO_STANDBY_CLUSTER) { + if (SS_REPLICATION_STANDBY_CLUSTER) { CreateRestartPoint(CHECKPOINT_IS_SHUTDOWN | CHECKPOINT_IMMEDIATE); } else { CreateCheckPoint(CHECKPOINT_IS_SHUTDOWN | CHECKPOINT_IMMEDIATE); @@ -18668,7 +18754,10 @@ int WriteXlogToShareStorage(XLogRecPtr startLsn, char *buf, int writeLen) pg_crc32c CalShareStorageCtlInfoCrc(const ShareStorageXLogCtl *ctlInfo) { - Assert(g_instance.xlog_cxt.shareStorageopCtl.isInit); + if (IS_SHARED_STORAGE_MODE) { + Assert(g_instance.xlog_cxt.shareStorageopCtl.isInit); + } + pg_crc32c crc; INIT_CRC32C(crc); COMP_CRC32C(crc, (char*)ctlInfo, offsetof(ShareStorageXLogCtl, crc)); @@ -18737,44 +18826,85 @@ void UpdatePostgresqlFile(const char *optName, const char *gucLine) release_file_lock(&filelock); } -void ShareStorageInit() +void SSWriteDoradoCtlInfoFile(int fd, char* buffer) { - if (g_instance.attr.attr_storage.xlog_file_path != NULL && g_instance.attr.attr_storage.xlog_file_size > 0) { - bool found = false; - void *tmpBuf = ShmemInitStruct("share storage Ctl", CalShareStorageCtlSize(), &found); - g_instance.xlog_cxt.shareStorageXLogCtl = (ShareStorageXLogCtl *)TYPEALIGN(MEMORY_ALIGNED_SIZE, tmpBuf); - g_instance.xlog_cxt.shareStorageXLogCtlOrigin = tmpBuf; - InitDoradoStorage(g_instance.attr.attr_storage.xlog_file_path, - (uint64)g_instance.attr.attr_storage.xlog_file_size); - if (!IsInitdb && g_instance.attr.attr_storage.xlog_lock_file_path != NULL) { - g_instance.xlog_cxt.shareStorageLockFd = BasicOpenFile(g_instance.attr.attr_storage.xlog_lock_file_path, - O_CREAT | O_RDWR | PG_BINARY, S_IRUSR | S_IWUSR); - if (g_instance.xlog_cxt.shareStorageLockFd < 0) { - ereport(ERROR, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", - g_instance.attr.attr_storage.xlog_lock_file_path))); - } - } + Assert(ENABLE_DSS && SS_CLUSTER_DORADO_REPLICATION); + Assert(fd > 0); + errno = EOK; + if (pwrite(fd, buffer, SS_DORADO_CTL_INFO_SIZE, 0) != SS_DORADO_CTL_INFO_SIZE) { + if (errno == 0) { + errno = ENOSPC; + } + ereport(PANIC, (errcode_for_file_access(), errmsg("could not write to ss ctl info file: %m"))); + } + + if (pg_fsync(fd) != 0) { + ereport(PANIC, (errcode_for_file_access(), errmsg("could not fsync ss ctl info file: %m"))); + } +} - if (((uint64)g_instance.attr.attr_storage.xlog_file_size != - g_instance.xlog_cxt.shareStorageopCtl.xlogFileSize) && (IS_SHARED_STORAGE_STANDBY_CLUSTER)) { - uint32 const bufSz = 256; - char buf[bufSz]; +void SSReadDoradoCtlInfoFile() +{ + Assert(ENABLE_DSS && SS_CLUSTER_DORADO_REPLICATION); + int fd = BasicOpenFile(SS_DORADO_CTRL_FILE, O_RDWR | PG_BINARY | O_DIRECT, S_IRUSR | S_IWUSR); + if (fd < 0) { + ereport(FATAL, (errcode_for_file_access(), errmsg("could not open ss ctl into file \"%s\": %m", SS_DORADO_CTRL_FILE))); + } + int readSize = CalShareStorageCtlSize(); + char buffer[readSize] __attribute__ ((__aligned__(ALIGNOF_BUFFER))); - errno_t errorno = snprintf_s(buf, sizeof(buf), sizeof(buf) - 1, "%lu", - g_instance.xlog_cxt.shareStorageopCtl.xlogFileSize); - securec_check_ss(errorno, "", ""); - SetConfigOption("xlog_file_size", buf, PGC_POSTMASTER, PGC_S_ARGV); - char option[bufSz]; - errorno = snprintf_s(option, sizeof(option), sizeof(option) - 1, "xlog_file_size=%lu\n", - g_instance.xlog_cxt.shareStorageopCtl.xlogFileSize); - securec_check_ss(errorno, "", ""); - UpdatePostgresqlFile("xlog_file_size", option); + if (pread(fd, buffer, SS_DORADO_CTL_INFO_SIZE, 0) != SS_DORADO_CTL_INFO_SIZE) { + ereport(PANIC, (errcode_for_file_access(), errmsg("could not read ss ctl into file: %m"))); + } + +} + +void NormalClusterDoradoStorageInit() +{ + if (g_instance.attr.attr_storage.xlog_file_path == NULL || g_instance.attr.attr_storage.xlog_file_size <= 0) { + return; + } + + bool found = false; + void *tmpBuf = ShmemInitStruct("share storage Ctl", CalShareStorageCtlSize(), &found); + g_instance.xlog_cxt.shareStorageXLogCtl = (ShareStorageXLogCtl *)TYPEALIGN(MEMORY_ALIGNED_SIZE, tmpBuf); + g_instance.xlog_cxt.shareStorageXLogCtlOrigin = tmpBuf; + InitDoradoStorage(g_instance.attr.attr_storage.xlog_file_path, + (uint64)g_instance.attr.attr_storage.xlog_file_size); + + if (!IsInitdb && g_instance.attr.attr_storage.xlog_lock_file_path != NULL) { + g_instance.xlog_cxt.shareStorageLockFd = BasicOpenFile(g_instance.attr.attr_storage.xlog_lock_file_path, + O_CREAT | O_RDWR | PG_BINARY, S_IRUSR | S_IWUSR); + if (g_instance.xlog_cxt.shareStorageLockFd < 0) { + ereport(ERROR, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", + g_instance.attr.attr_storage.xlog_lock_file_path))); } if (ENABLE_DMS) { g_instance.dms_cxt.SSRecoveryInfo.dorado_sharestorage_inited = true; } } + + if (((uint64)g_instance.attr.attr_storage.xlog_file_size != + g_instance.xlog_cxt.shareStorageopCtl.xlogFileSize) && (IS_SHARED_STORAGE_STANDBY_CLUSTER)) { + char buf[MAX_XLOG_FILE_SIZE_BUFFER]; + + errno_t errorno = snprintf_s(buf, sizeof(buf), sizeof(buf) - 1, "%lu", + g_instance.xlog_cxt.shareStorageopCtl.xlogFileSize); + securec_check_ss(errorno, "", ""); + SetConfigOption("xlog_file_size", buf, PGC_POSTMASTER, PGC_S_ARGV); + char option[MAX_XLOG_FILE_SIZE_BUFFER]; + errorno = snprintf_s(option, sizeof(option), sizeof(option) - 1, "xlog_file_size=%lu\n", + g_instance.xlog_cxt.shareStorageopCtl.xlogFileSize); + securec_check_ss(errorno, "", ""); + UpdatePostgresqlFile("xlog_file_size", option); + } +} + +void ShareStorageInit() +{ + NormalClusterDoradoStorageInit(); + SSClusterDoradoStorageInit(); } void ShareStorageSetBuildErrorAndExit(HaRebuildReason reason, bool setRcvDone) @@ -19104,16 +19234,26 @@ static int SSReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int XLByteToSeg(targetPagePtr, t_thrd.xlog_cxt.readSegNo); XLByteAdvance(RecPtr, reqLen); + /* get insertHead of ctl_info file */ + if (SS_CLUSTER_DORADO_REPLICATION) { + ReadSSDoradoCtlInfoFile(); + ShareStorageXLogCtl *ctlInfo = g_instance.xlog_cxt.ssReplicationXLogCtl; + t_thrd.xlog_cxt.receivedUpto = ctlInfo->insertHead; + } + retry: /* See if we need to retrieve more data */ + /* In ss dorado replication, we don't start walrecwrite thread, so t_thrd.xlog_cxt.receivedUpto = 0 */ if (t_thrd.xlog_cxt.readFile < 0 || (t_thrd.xlog_cxt.readSource == XLOG_FROM_STREAM && XLByteLT(t_thrd.xlog_cxt.receivedUpto, RecPtr))) { - if (t_thrd.xlog_cxt.StandbyMode && t_thrd.xlog_cxt.startup_processing && DORADO_STANDBY_CLUSTER) { + if (t_thrd.xlog_cxt.StandbyMode && t_thrd.xlog_cxt.startup_processing && (DORADO_STANDBY_CLUSTER || SS_CLUSTER_DORADO_REPLICATION)) { + /* * In standby mode, wait for the requested record to become * available, either via restore_command succeeding to restore the * segment, or via walreceiver having streamed the record. */ + for (;;) { /* * Need to check here also for the case where consistency level is @@ -19174,18 +19314,26 @@ static int SSReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int if (XLByteLT(expectedRecPtr, t_thrd.xlog_cxt.receivedUpto)) { havedata = true; } else { + havedata = false; XLogRecPtr latestChunkStart; + if (SS_CLUSTER_DORADO_REPLICATION) { + ReadSSDoradoCtlInfoFile(); + t_thrd.xlog_cxt.receivedUpto = g_instance.xlog_cxt.ssReplicationXLogCtl->insertHead; + latestChunkStart = g_instance.xlog_cxt.ssReplicationXLogCtl->insertHead; + } else { + t_thrd.xlog_cxt.receivedUpto = GetWalRcvWriteRecPtr(&latestChunkStart); + } - t_thrd.xlog_cxt.receivedUpto = GetWalRcvWriteRecPtr(&latestChunkStart); if (XLByteLT(expectedRecPtr, t_thrd.xlog_cxt.receivedUpto)) { - havedata = true; - if (!XLByteLT(RecPtr, latestChunkStart)) { - t_thrd.xlog_cxt.XLogReceiptTime = GetCurrentTimestamp(); - SetCurrentChunkStartTime(t_thrd.xlog_cxt.XLogReceiptTime); + havedata = true; + if (!XLByteLT(RecPtr, latestChunkStart)) { + t_thrd.xlog_cxt.XLogReceiptTime = GetCurrentTimestamp(); + SetCurrentChunkStartTime(t_thrd.xlog_cxt.XLogReceiptTime); + } + } else { + havedata = false; } - } else { - havedata = false; - } + } if (havedata) { /* @@ -19211,7 +19359,7 @@ static int SSReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int t_thrd.xlog_cxt.RedoDone = IsRedoDonePromoting(); pg_memory_barrier(); - if (IS_SHARED_STORAGE_MODE) { + if (IS_SHARED_STORAGE_MODE || SS_CLUSTER_DORADO_REPLICATION) { uint32 disableConnectionNode = pg_atomic_read_u32(&g_instance.comm_cxt.localinfo_cxt.need_disable_connection_node); if (disableConnectionNode && WalRcvIsRunning()) { @@ -19296,7 +19444,7 @@ static int SSReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int ProcTxnWorkLoad(true); if (!xlogctl->IsRecoveryDone) { SendPostmasterSignal(PMSIGNAL_LOCAL_RECOVERY_DONE); - if (DORADO_STANDBY_CLUSTER && SS_PERFORMING_SWITCHOVER) { + if ((DORADO_STANDBY_CLUSTER || SS_REPLICATION_STANDBY_CLUSTER) && SS_PERFORMING_SWITCHOVER) { g_instance.dms_cxt.SSClusterState = NODESTATE_STANDBY_PROMOTED; } } @@ -19338,7 +19486,7 @@ static int SSReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int */ load_server_mode(); - if (DORADO_STANDBY_CLUSTER_MAINSTANDBY_NODE) { + if (DORADO_STANDBY_CLUSTER_MAINSTANDBY_NODE || IS_SS_REPLICATION_MAIN_STANBY_NODE) { ProcTxnWorkLoad(false); /* use volatile pointer to prevent code rearrangement */ volatile WalRcvData *walrcv = t_thrd.walreceiverfuncs_cxt.WalRcv; @@ -19428,7 +19576,7 @@ static int SSReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int /* Read the requested page */ t_thrd.xlog_cxt.readOff = targetPageOff; - if (DORADO_STANDBY_CLUSTER_MAINSTANDBY_NODE) { + if (DORADO_STANDBY_CLUSTER_MAINSTANDBY_NODE || IS_SS_REPLICATION_MAIN_STANBY_NODE) { try_again: if (lseek(t_thrd.xlog_cxt.readFile, (off_t)t_thrd.xlog_cxt.readOff, SEEK_SET) < 0) { ereport(emode_for_corrupt_record(emode, RecPtr), diff --git a/src/gausskernel/storage/dorado_operation/dorado_fd.cpp b/src/gausskernel/storage/dorado_operation/dorado_fd.cpp index 224df812c..e44d9575b 100644 --- a/src/gausskernel/storage/dorado_operation/dorado_fd.cpp +++ b/src/gausskernel/storage/dorado_operation/dorado_fd.cpp @@ -25,8 +25,8 @@ #include "knl/knl_variable.h" #include "access/xlog.h" #include "storage/dorado_operation/dorado_fd.h" +#include "miscadmin.h" -const uint32 DORADO_CTL_WRITE_SIZE = 512; const uint32 DORADO_XLOG_START_POS = XLogSegSize; void DoradoWriteCtlInfo(const ShareStorageXLogCtl *ctlInfo); @@ -112,7 +112,8 @@ void DoradoReadCtlInfo(ShareStorageXLogCtl *ctlInfo) } if (ctlInfo->magic != SHARE_STORAGE_CTL_MAGIC || ctlInfo->checkNumber != SHARE_STORAGE_CTL_CHCK_NUMBER) { - ereport(FATAL, (errmsg("dorado ctl info maybe damaged"))); + ereport(FATAL, (errmsg("dorado ctl info maybe damaged, cltInfo->magic = %u, ctlInfo->checkNumber = %llu", + ctlInfo->magic, ctlInfo->checkNumber))); } pg_crc32c crc = CalShareStorageCtlInfoCrc(ctlInfo); diff --git a/src/gausskernel/storage/ipc/procarray.cpp b/src/gausskernel/storage/ipc/procarray.cpp index 7511bcac8..ae0199f6c 100755 --- a/src/gausskernel/storage/ipc/procarray.cpp +++ b/src/gausskernel/storage/ipc/procarray.cpp @@ -122,6 +122,7 @@ #include "gstrace/storage_gstrace.h" #include "ddes/dms/ss_common_attr.h" #include "ddes/dms/ss_transaction.h" +#include "replication/ss_cluster_replication.h" #ifdef ENABLE_UT #define static @@ -2362,12 +2363,12 @@ Snapshot GetSnapshotData(Snapshot snapshot, bool force_local_snapshot) } /* Check whether there's a standby requiring an older xmin when dms is enabled. */ - if (ENABLE_DMS && SS_STANDBY_CLUSTER_NORMAL_MAIN_STANDBY) { + if (ENABLE_DMS && (SS_STANDBY_CLUSTER_NORMAL_MAIN_STANDBY || (SS_NORMAL_PRIMARY && IS_SS_REPLICATION_MAIN_STANBY_NODE))) { ss_xmin_info_t* xmin_info = &g_instance.dms_cxt.SSXminInfo; uint64 global_xmin = SSGetGlobalOldestXmin(u_sess->utils_cxt.RecentGlobalXmin); u_sess->utils_cxt.RecentGlobalXmin = global_xmin; } - + /* Non-catalog tables can be vacuumed if older than this xid */ u_sess->utils_cxt.RecentGlobalDataXmin = u_sess->utils_cxt.RecentGlobalXmin; diff --git a/src/gausskernel/storage/replication/CMakeLists.txt b/src/gausskernel/storage/replication/CMakeLists.txt index a33849974..ae93af7e9 100755 --- a/src/gausskernel/storage/replication/CMakeLists.txt +++ b/src/gausskernel/storage/replication/CMakeLists.txt @@ -63,6 +63,7 @@ set(TGT_replication_SRC ${CMAKE_CURRENT_SOURCE_DIR}/shared_storage_walreceiver.cpp ${CMAKE_CURRENT_SOURCE_DIR}/subscription_walreceiver.cpp ${CMAKE_CURRENT_SOURCE_DIR}/libpqsw.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/ss_cluster_replication.cpp ) set(TGT_replication_INC diff --git a/src/gausskernel/storage/replication/Makefile b/src/gausskernel/storage/replication/Makefile index b28fa1138..89da79b60 100644 --- a/src/gausskernel/storage/replication/Makefile +++ b/src/gausskernel/storage/replication/Makefile @@ -16,7 +16,7 @@ endif OBJS = walsender.o datasender.o walreceiverfuncs.o walreceiver.o walrcvwriter.o subscription_walreceiver.o\ datareceiver.o datarcvwriter.o basebackup.o libpqwalreceiver.o archive_walreceiver.o repl_gram.o\ syncrep.o dataqueue.o bcm.o datasyncrep.o catchup.o slot.o slotfuncs.o shared_storage_walreceiver.o\ - syncrep_gram.o heartbeat.o rto_statistic.o libpqsw.o + syncrep_gram.o heartbeat.o rto_statistic.o libpqsw.o ss_cluster_replication.o SUBDIRS = logical heartbeat dcf include $(top_srcdir)/src/gausskernel/common.mk diff --git a/src/gausskernel/storage/replication/basebackup.cpp b/src/gausskernel/storage/replication/basebackup.cpp index fc802fc9e..6aebca5d5 100755 --- a/src/gausskernel/storage/replication/basebackup.cpp +++ b/src/gausskernel/storage/replication/basebackup.cpp @@ -1526,8 +1526,10 @@ static int64 sendDir(const char *path, int basepathlen, bool sizeonly, List *tab * WAL archive anyway. But include it as an empty directory anyway, so * we get permissions right. */ + + /* when ss dorado replication enabled, "+data/pg_replication/" also need to copy when backup */ int pathNameLen = strlen("+data/pg_xlog"); - if (strcmp(pathbuf, "./pg_xlog") == 0 || strncmp(pathbuf, "+data/pg_xlog", pathNameLen) == 0) { + if (strcmp(pathbuf, "./pg_xlog") == 0 || strncmp(pathbuf, "+data/pg_xlog", pathNameLen) == 0 || strcmp(pathbuf, "+data/pg_replication") == 0) { if (!sizeonly) { /* If pg_xlog is a symlink, write it as a directory anyway */ #ifndef WIN32 diff --git a/src/gausskernel/storage/replication/dataqueue.cpp b/src/gausskernel/storage/replication/dataqueue.cpp index 19e53540d..4424a1e2e 100644 --- a/src/gausskernel/storage/replication/dataqueue.cpp +++ b/src/gausskernel/storage/replication/dataqueue.cpp @@ -37,6 +37,7 @@ #include "replication/syncrep.h" #include "replication/walsender.h" #include "replication/walreceiver.h" +#include "replication/ss_cluster_replication.h" #include "storage/lock/lwlock.h" #include "storage/proc.h" #include "storage/shmem.h" @@ -300,7 +301,7 @@ DataQueuePtr PushToSenderQueue(const RelFileNode &rnode, BlockNumber blockNum, S LWLockRelease(DataSyncRepLock); if (g_instance.attr.attr_storage.max_wal_senders > 0) { - if (t_thrd.walsender_cxt.WalSndCtl->sync_master_standalone && !IS_SHARED_STORAGE_MODE) { + if (t_thrd.walsender_cxt.WalSndCtl->sync_master_standalone && !(IS_SHARED_STORAGE_MODE || SS_CLUSTER_DORADO_REPLICATION)) { ereport( LOG, (errmsg("failed to push rnode %u/%u/%u blockno %u into data-queue becuase sync_master_standalone " diff --git a/src/gausskernel/storage/replication/datasyncrep.cpp b/src/gausskernel/storage/replication/datasyncrep.cpp index ff3e85b69..50aef20d9 100644 --- a/src/gausskernel/storage/replication/datasyncrep.cpp +++ b/src/gausskernel/storage/replication/datasyncrep.cpp @@ -34,6 +34,7 @@ #include "replication/datasender_private.h" #include "replication/walsender_private.h" #include "replication/shared_storage_walreceiver.h" +#include "replication/ss_cluster_replication.h" #include "replication/syncrep.h" #include "storage/cu.h" #include "storage/pmsignal.h" @@ -185,7 +186,7 @@ void WaitForDataSync(void) /* * if we modify the syncmode dynamically, we'll stop wait */ - if ((t_thrd.walsender_cxt.WalSndCtl->sync_master_standalone && !IS_SHARED_STORAGE_MODE) || + if ((t_thrd.walsender_cxt.WalSndCtl->sync_master_standalone && !(IS_SHARED_STORAGE_MODE || SS_CLUSTER_DORADO_REPLICATION)) || u_sess->attr.attr_storage.guc_synchronous_commit <= SYNCHRONOUS_COMMIT_LOCAL_FLUSH) { ereport(WARNING, (errmsg("canceling wait for synchronous replication due to syncmaster standalone."), diff --git a/src/gausskernel/storage/replication/libpqwalreceiver.cpp b/src/gausskernel/storage/replication/libpqwalreceiver.cpp index 1b08d5ea5..d211663fc 100755 --- a/src/gausskernel/storage/replication/libpqwalreceiver.cpp +++ b/src/gausskernel/storage/replication/libpqwalreceiver.cpp @@ -28,6 +28,7 @@ #include "replication/walreceiver.h" #include "replication/walsender_private.h" #include "replication/libpqwalreceiver.h" +#include "replication/ss_cluster_replication.h" #include "storage/pmsignal.h" #include "storage/proc.h" #include "utils/guc.h" @@ -236,6 +237,21 @@ static bool CheckRemoteServerSharedStorage(ServerMode remoteMode, PGresult* res) return true; } +static bool CheckSSRemoteServerMode(ServerMode remoteMode, PGresult* res) +{ + if (IS_SS_REPLICATION_MAIN_STANBY_NODE) { + if (remoteMode != PRIMARY_MODE) { + PQclear(res); + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("when ss replication, main standby of standby cluster of the remote server must be primary, current is %s", + wal_get_role_string(remoteMode, true)))); + return false; + } + } + + return true; +} + void StartRemoteStreaming(const LibpqrcvConnectParam *options) { Assert(t_thrd.libwalreceiver_cxt.streamConn != NULL); @@ -477,7 +493,7 @@ ServerMode IdentifyRemoteMode() !t_thrd.walreceiver_cxt.AmWalReceiverForFailover && (!IS_PRIMARY_NORMAL(remoteMode)) && /* remoteMode of cascade standby is a standby */ - !t_thrd.xlog_cxt.is_cascade_standby && !IS_SHARED_STORAGE_MODE) { + !t_thrd.xlog_cxt.is_cascade_standby && !(IS_SHARED_STORAGE_MODE || SS_CLUSTER_DORADO_REPLICATION)) { PQclear(res); if (dummyStandbyMode) { @@ -490,7 +506,7 @@ ServerMode IdentifyRemoteMode() } if (t_thrd.postmaster_cxt.HaShmData->is_cascade_standby && remoteMode != STANDBY_MODE && - !IS_SHARED_STORAGE_MODE) { + !(IS_SHARED_STORAGE_MODE || SS_CLUSTER_DORADO_REPLICATION)) { PQclear(res); SpinLockAcquire(&walrcv->mutex); @@ -508,6 +524,10 @@ ServerMode IdentifyRemoteMode() } } + if (SS_CLUSTER_DORADO_REPLICATION && CheckSSRemoteServerMode(remoteMode, res)) { + return UNKNOWN_MODE; + } + PQclear(res); return remoteMode; } @@ -1395,7 +1415,8 @@ bool libpqrcv_receive(int timeout, unsigned char *type, char **buffer, int *len) return true; } *type = *((unsigned char *)t_thrd.libwalreceiver_cxt.recvBuf); - if (IS_SHARED_STORAGE_MODE && !AM_HADR_WAL_RECEIVER && *type == 'w') { + + if ((IS_SHARED_STORAGE_MODE || SS_CLUSTER_DORADO_REPLICATION) && !AM_HADR_WAL_RECEIVER && *type == 'w') { *len = 0; return false; } diff --git a/src/gausskernel/storage/replication/shared_storage_walreceiver.cpp b/src/gausskernel/storage/replication/shared_storage_walreceiver.cpp index 7e3af8dad..a166f4caa 100644 --- a/src/gausskernel/storage/replication/shared_storage_walreceiver.cpp +++ b/src/gausskernel/storage/replication/shared_storage_walreceiver.cpp @@ -30,6 +30,7 @@ #include "miscadmin.h" #include "replication/walreceiver.h" #include "replication/shared_storage_walreceiver.h" +#include "replication/ss_cluster_replication.h" #include "replication/slot.h" #include "storage/pmsignal.h" #include "storage/proc.h" @@ -367,28 +368,36 @@ bool shared_storage_connect(char *conninfo, XLogRecPtr *startpoint, char *slotna walrcv->peer_role = PRIMARY_MODE; walrcv->peer_state = NORMAL_STATE; walrcv->isFirstTimeAccessStorage = true; - uint32 totalLen = sizeof(WalDataMessageHeader) + ShareStorageBufSize + - g_instance.xlog_cxt.shareStorageopCtl.blkSize; - t_thrd.libwalreceiver_cxt.shared_storage_buf = (char *)palloc0(totalLen); - t_thrd.libwalreceiver_cxt.shared_storage_read_buf = - (char *)TYPEALIGN(g_instance.xlog_cxt.shareStorageopCtl.blkSize, - t_thrd.libwalreceiver_cxt.shared_storage_buf + sizeof(WalDataMessageHeader)); - t_thrd.libwalreceiver_cxt.connect_param.conninfo = conninfo; - t_thrd.libwalreceiver_cxt.connect_param.startpoint = *startpoint; - t_thrd.libwalreceiver_cxt.connect_param.slotname = slotname; - t_thrd.libwalreceiver_cxt.connect_param.channel_identifier = channel_identifier; - XLogReaderState *xlogreader = XLogReaderAllocate(SharedStorageXLogPageRead, 0, - g_instance.xlog_cxt.shareStorageopCtl.blkSize); - if (xlogreader == NULL) { - ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"), - errdetail("Failed while allocating an XLog reading processor"))); + + if (SS_CLUSTER_DORADO_REPLICATION) { + libpgConnected = libpqrcv_connect(conninfo, startpoint, slotname, channel_identifier); + return libpgConnected; } - xlogreader->system_identifier = GetSystemIdentifier(); - t_thrd.libwalreceiver_cxt.xlogreader = xlogreader; - shared_storage_xlog_check_consistency(); + if (IS_SHARED_STORAGE_MODE) { + uint32 totalLen = sizeof(WalDataMessageHeader) + ShareStorageBufSize + + g_instance.xlog_cxt.shareStorageopCtl.blkSize; + t_thrd.libwalreceiver_cxt.shared_storage_buf = (char *)palloc0(totalLen); + t_thrd.libwalreceiver_cxt.shared_storage_read_buf = + (char *)TYPEALIGN(g_instance.xlog_cxt.shareStorageopCtl.blkSize, + t_thrd.libwalreceiver_cxt.shared_storage_buf + sizeof(WalDataMessageHeader)); + t_thrd.libwalreceiver_cxt.connect_param.conninfo = conninfo; + t_thrd.libwalreceiver_cxt.connect_param.startpoint = *startpoint; + t_thrd.libwalreceiver_cxt.connect_param.slotname = slotname; + t_thrd.libwalreceiver_cxt.connect_param.channel_identifier = channel_identifier; + XLogReaderState *xlogreader = XLogReaderAllocate(SharedStorageXLogPageRead, 0, + g_instance.xlog_cxt.shareStorageopCtl.blkSize); + if (xlogreader == NULL) { + ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"), + errdetail("Failed while allocating an XLog reading processor"))); + } + xlogreader->system_identifier = GetSystemIdentifier(); + t_thrd.libwalreceiver_cxt.xlogreader = xlogreader; - libpgConnected = try_connect_libpq(&t_thrd.libwalreceiver_cxt.connect_param, true); + shared_storage_xlog_check_consistency(); + + libpgConnected = try_connect_libpq(&t_thrd.libwalreceiver_cxt.connect_param, true); + } return libpgConnected; } @@ -425,12 +434,20 @@ bool shared_storage_receive(int timeout, unsigned char *type, char **buffer, int } } + /* + * When ss cluster replication enabled, no xlog will receive, so return false directly. + * Xlog will replicated by Dorado synchronous replication. + */ + if (SS_CLUSTER_DORADO_REPLICATION) { + return false; + } + return shared_storage_xlog_read(timeout / timeRatio, type, buffer, len, isStopping); } void shared_storage_send(const char *buffer, int nbytes) { - if (IS_SHARED_STORAGE_STANBY_MODE) { + if (IS_SHARED_STORAGE_STANBY_MODE || SS_CLUSTER_DORADO_REPLICATION) { if (t_thrd.libwalreceiver_cxt.streamConn) libpqrcv_send(buffer, nbytes); } diff --git a/src/gausskernel/storage/replication/ss_cluster_replication.cpp b/src/gausskernel/storage/replication/ss_cluster_replication.cpp new file mode 100644 index 000000000..3283018b9 --- /dev/null +++ b/src/gausskernel/storage/replication/ss_cluster_replication.cpp @@ -0,0 +1,264 @@ +/* + * Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * Portions Copyright (c) 2021, openGauss Contributors + * + * Description: openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * --------------------------------------------------------------------------------------- + * + * + * + * IDENTIFICATION + * src/gausskernel/storage/replication/ss_cluster_replication.cpp + * + * --------------------------------------------------------------------------------------- + */ + +#include "replication/ss_cluster_replication.h" +#include "access/xlog_internal.h" +#include "storage/file/fio_device.h" +#include "storage/smgr/fd.h" + + +void WriteSSDoradoCtlInfoFile() +{ + Assert(dss_exist_file(SS_DORADO_CTRL_FILE)); + Assert(g_instance.xlog_cxt.ssReplicationXLogCtl != NULL); + ShareStorageXLogCtl *ctlInfo = g_instance.xlog_cxt.ssReplicationXLogCtl; + errno_t errorno = EOK; + + int fd = BasicOpenFile(SS_DORADO_CTRL_FILE, O_RDWR | PG_BINARY, S_IRUSR | S_IWUSR); + if (fd < 0) { + ereport(PANIC, + (errcode_for_file_access(), errmsg("[WriteSSDoradoCtlInfoFile]could not open SS dorado control file \"%s\".", + SS_DORADO_CTRL_FILE))); + } + + pg_crc32c crc = CalShareStorageCtlInfoCrc(ctlInfo); + ctlInfo->crc = crc; + + char buffer[SS_DORADO_CTL_INFO_SIZE] __attribute__((__aligned__(ALIGNOF_BUFFER))); + + errorno = memcpy_s(buffer, SS_DORADO_CTL_INFO_SIZE, ctlInfo, SS_DORADO_CTL_INFO_SIZE); + securec_check_c(errorno, "\0", "\0"); + + if (write(fd, buffer, SS_DORADO_CTL_INFO_SIZE) != SS_DORADO_CTL_INFO_SIZE) { + ereport(PANIC, + (errcode_for_file_access(), errmsg("[WriteSSDoradoCtlInfoFile]could not write SS dorado control file \"%s\".", + SS_DORADO_CTRL_FILE))); + } + + if (close(fd)) { + ereport(PANIC, + (errcode_for_file_access(), errmsg("[WriteSSDoradoCtlInfoFile]could not close SS dorado control file \"%s\".", + SS_DORADO_CTRL_FILE))); + } +} + +void ReadSSDoradoCtlInfoFile() +{ + Assert(dss_exist_file(SS_DORADO_CTRL_FILE)); + Assert(g_instance.xlog_cxt.ssReplicationXLogCtl != NULL); + ShareStorageXLogCtl *ctlInfo = g_instance.xlog_cxt.ssReplicationXLogCtl; + errno_t errorno = EOK; + int fd = -1; + fd = BasicOpenFile(SS_DORADO_CTRL_FILE, O_RDWR | PG_BINARY, S_IRUSR | S_IWUSR); + if (fd < 0) { + /* TODO: need consider that dorado is briefly unreadable during the synchronization process */ + ereport(PANIC, + (errcode_for_file_access(), errmsg("[ReadSSDoradoCtlInfo]could not create SS dorado control file \"%s\".", + SS_DORADO_CTRL_FILE))); + } + + char buffer[SS_DORADO_CTL_INFO_SIZE] __attribute__((__aligned__(ALIGNOF_BUFFER))); + if (read(fd, buffer, SS_DORADO_CTL_INFO_SIZE) != SS_DORADO_CTL_INFO_SIZE) { + ereport(PANIC, + (errcode_for_file_access(), errmsg("[ReadSSDoradoCtlInfo]could not read SS dorado control file \"%s\".", + SS_DORADO_CTRL_FILE))); + } + + errorno = memcpy_s(ctlInfo, SS_DORADO_CTL_INFO_SIZE, buffer, SS_DORADO_CTL_INFO_SIZE); + securec_check_c(errorno, "\0", "\0"); + if (close(fd)) { + ereport(PANIC, + (errcode_for_file_access(), errmsg("[ReadSSDoradoCtlInfo]could not close SS dorado control file \"%s\".", + SS_DORADO_CTRL_FILE))); + } + + if ((ctlInfo->magic != SHARE_STORAGE_CTL_MAGIC) || (ctlInfo->checkNumber != SHARE_STORAGE_CTL_CHCK_NUMBER)) { + ereport(FATAL, (errmsg("[ReadSSDoradoCtlInfo]SS replication ctl_info maybe damaged."))); + } + + pg_crc32c crc = CalShareStorageCtlInfoCrc(ctlInfo); + if (!EQ_CRC32C(crc, ctlInfo->crc)) { + ereport(FATAL, (errmsg("[ReadSSDoradoCtlInfo]SS replication ctl_info crc check failed."))); + } +} + +void InitSSDoradoCtlInfoFile() +{ + if (dss_exist_file(SS_DORADO_CTRL_FILE)) { + ReadSSDoradoCtlInfoFile(); + ereport(LOG, (errcode_for_file_access(), errmsg("[InitSSDoradoCtlInfoFile] Dorado ctl info file already exists."))); + return; + } + + ShareStorageXLogCtl *ctlInfo = g_instance.xlog_cxt.ssReplicationXLogCtl; + Assert(ctlInfo != NULL); + + int fd = -1; + char buffer[SS_DORADO_CTL_INFO_SIZE] __attribute__((__aligned__(ALIGNOF_BUFFER))); /* need to be aligned */ + errno_t errorno = EOK; + Assert(!dss_exist_file(SS_DORADO_CTRL_FILE)); + + /* create SS_DORADO_CTRL_FILE first time */ + fd = BasicOpenFile(SS_DORADO_CTRL_FILE, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, S_IRUSR | S_IWUSR); + if (fd < 0) { + ereport(PANIC, + (errcode_for_file_access(), errmsg("could not create SS dorado control file \"%s\".", SS_DORADO_CTRL_FILE))); + } + + ereport(LOG, (errcode_for_file_access(), errmsg("[InitSSDoradoCtlInfoFile] Create SS dorado ctl info succ."))); + + InitSSDoradoCtlInfo(ctlInfo, 0); + ctlInfo->crc = CalShareStorageCtlInfoCrc(ctlInfo); + + Assert(sizeof(ShareStorageXLogCtl) > SS_DORADO_CTL_INFO_SIZE); + errorno = memcpy_s(buffer, SS_DORADO_CTL_INFO_SIZE, ctlInfo, SS_DORADO_CTL_INFO_SIZE); + securec_check_c(errorno, "\0", "\0"); + if (write(fd, buffer, SS_DORADO_CTL_INFO_SIZE) != SS_DORADO_CTL_INFO_SIZE) { + ereport(PANIC, + (errcode_for_file_access(), errmsg("could not write SS dorado control file \"%s\".", SS_DORADO_CTRL_FILE))); + } + + if (pg_fsync(fd) != 0) { + ereport(PANIC, + (errcode_for_file_access(), errmsg("could not fsync SS dorado control file \"%s\".", SS_DORADO_CTRL_FILE))); + } + if (close(fd)) { + ereport(PANIC, + (errcode_for_file_access(), errmsg("could not close SS dorado control file \"%s\".", SS_DORADO_CTRL_FILE))); + } +} + +void SSClusterDoradoStorageInit() +{ + if (!SS_CLUSTER_DORADO_REPLICATION) { + return; + } + + bool found = false; + g_instance.xlog_cxt.ssReplicationXLogCtl = (ShareStorageXLogCtl*)ShmemInitStruct("SS Replication Xlog Ctl", sizeof(ShareStorageXLogCtl), &found); + InitSSDoradoCtlInfoFile(); + + if (!IsInitdb && ENABLE_DMS) { + g_instance.dms_cxt.SSRecoveryInfo.dorado_sharestorage_inited = true; + } +} + +void UpdateSSDoradoCtlInfoAndSync() +{ + if (!SS_CLUSTER_DORADO_REPLICATION) { + return; + } + + ReadSSDoradoCtlInfoFile(); + ShareStorageXLogCtl *ctlInfo = g_instance.xlog_cxt.ssReplicationXLogCtl; + ctlInfo->insertHead = t_thrd.xlog_cxt.LogwrtResult->Write; + WriteSSDoradoCtlInfoFile(); +} + +bool CheckSSCtlInfoConsistency(XLogRecPtr localEnd) +{ + if (!SS_CLUSTER_DORADO_REPLICATION) { + return true; + } + + ReadSSDoradoCtlInfoFile(); + ShareStorageXLogCtl *ctlInfo = g_instance.xlog_cxt.ssReplicationXLogCtl; + + if (ctlInfo->systemIdentifier != GetSystemIdentifier()) { + ereport(FATAL, (errmsg("database system version is different between shared storage %lu and local %lu", + ctlInfo->systemIdentifier, GetSystemIdentifier()))); + } + + return true; +} + +void InitSSDoradoCtlInfo(ShareStorageXLogCtl *ctlInfo, uint64 sysidentifier) +{ + ctlInfo->magic = SHARE_STORAGE_CTL_MAGIC; + ctlInfo->length = SizeOfShareStorageXLogCtl; + ctlInfo->version = CURRENT_SHARE_STORAGE_CTL_VERSION; + ctlInfo->systemIdentifier = sysidentifier; + ctlInfo->insertHead = 0; + ctlInfo->xlogFileSize = 0; + ctlInfo->checkNumber = SHARE_STORAGE_CTL_CHCK_NUMBER; + ctlInfo->insertTail = 0; + ctlInfo->term = 1; + ctlInfo->pad1 = 0; + ctlInfo->pad2 = 0; + ctlInfo->pad3 = 0; + ctlInfo->pad4 = 0; +} + +void CheckSSDoradoCtlInfo(XLogRecPtr localEnd) +{ + if (!SS_CLUSTER_DORADO_REPLICATION) { + return; + } + + ReadSSDoradoCtlInfoFile(); + ShareStorageXLogCtl *ctlInfo = g_instance.xlog_cxt.ssReplicationXLogCtl; + + if (ctlInfo->systemIdentifier != GetSystemIdentifier()) { + ereport(FATAL, (errmsg("database system version is different between shared storage %lu and local %lu", + ctlInfo->systemIdentifier, GetSystemIdentifier()))); + } + + uint32 shiftSize = 32; + if (localEnd < ctlInfo->insertHead) { + ereport(LOG, + (errmsg("modify insertHead from %X/%X to %X/%X", + static_cast(ctlInfo->insertHead >> shiftSize), static_cast(ctlInfo->insertHead), + static_cast(localEnd >> shiftSize), static_cast(localEnd)))); + ctlInfo->insertHead = localEnd; + WriteSSDoradoCtlInfoFile(); + } else { + XLogRecPtr shareEnd = ctlInfo->insertHead; + if (0 == shareEnd % XLogSegSize) { + XLByteAdvance(shareEnd, SizeOfXLogLongPHD); + } else if (0 == shareEnd % XLOG_BLCKSZ) { + XLByteAdvance(shareEnd, SizeOfXLogShortPHD); + } + if (XLByteLT(shareEnd, localEnd)) { + char path[MAXPGPATH]; + XLogSegNo sendSegNo; + XLByteToSeg(ctlInfo->insertHead, sendSegNo); + /* Need to consider other xlog path?? */ + XLogFilePath(path, MAXPGPATH, t_thrd.xlog_cxt.ThisTimeLineID, sendSegNo); + struct stat stat_buf; + + if (stat(path, &stat_buf) != 0) { + ereport(FATAL, (errmsg("the local's tail is bigger than ctlInfo insertHead %X/%X, path %s", + static_cast(ctlInfo->insertHead >> shiftSize), + static_cast(ctlInfo->insertHead), path))); + } + } + } + + uint32 localTerm = Max(g_instance.comm_cxt.localinfo_cxt.term_from_file, + g_instance.comm_cxt.localinfo_cxt.term_from_xlog); + if (localTerm > ctlInfo->term) { + ctlInfo->term = localTerm; + WriteSSDoradoCtlInfoFile(); + } +} \ No newline at end of file diff --git a/src/gausskernel/storage/replication/walreceiver.cpp b/src/gausskernel/storage/replication/walreceiver.cpp index efaf50ce5..3d8c4b770 100755 --- a/src/gausskernel/storage/replication/walreceiver.cpp +++ b/src/gausskernel/storage/replication/walreceiver.cpp @@ -59,6 +59,7 @@ #include "replication/walsender.h" #include "replication/walsender_private.h" #include "replication/dcf_replication.h" +#include "replication/ss_cluster_replication.h" #include "storage/copydir.h" #include "storage/ipc.h" #include "storage/latch.h" @@ -468,9 +469,9 @@ void WalReceiverMain(void) int nRet = 0; errno_t rc = 0; - if (ENABLE_DSS && t_thrd.postmaster_cxt.HaShmData->current_mode == STANDBY_MODE && + if ((ENABLE_DSS && t_thrd.postmaster_cxt.HaShmData->current_mode == STANDBY_MODE && g_instance.attr.attr_common.cluster_run_mode == RUN_MODE_STANDBY && - g_instance.attr.attr_storage.xlog_file_path != 0) { + g_instance.attr.attr_storage.xlog_file_path != 0) || IS_SS_REPLICATION_MAIN_STANBY_NODE) { ereport(LOG, (errmsg("walreceiver thread started for main standby"))); } else { Assert(ENABLE_DSS == false); @@ -847,6 +848,10 @@ bool HasBuildReason() static void rcvAllXlog() { + if (SS_CLUSTER_DORADO_REPLICATION) { + return; + } + if (HasBuildReason() || t_thrd.walreceiver_cxt.checkConsistencyOK == false) { ereport(LOG, (errmsg("rcvAllXlog no need copy xlog buildreason:%d, check flag:%d", (int)t_thrd.postmaster_cxt.HaShmData->repl_reason[t_thrd.postmaster_cxt.HaShmData->current_repl], @@ -884,7 +889,7 @@ static void WalRcvDie(int code, Datum arg) { /* use volatile pointer to prevent code rearrangement */ volatile WalRcvData *walrcv = t_thrd.walreceiverfuncs_cxt.WalRcv; - if (IS_SHARED_STORAGE_MODE && !t_thrd.walreceiver_cxt.termChanged) { + if ((IS_SHARED_STORAGE_MODE || SS_CLUSTER_DORADO_REPLICATION) && !t_thrd.walreceiver_cxt.termChanged) { SpinLockAcquire(&walrcv->mutex); walrcv->walRcvState = WALRCV_STOPPING; SpinLockRelease(&walrcv->mutex); @@ -1558,6 +1563,11 @@ void XLogWalRcvSendReply(bool force, bool requestReply) ReadShareStorageCtlInfo(ctlInfo); receivePtr = ctlInfo->insertHead; AlignFreeShareStorageCtl(ctlInfo); + } else if (SS_CLUSTER_DORADO_REPLICATION) { + ReadSSDoradoCtlInfoFile(); + receivePtr = g_instance.xlog_cxt.ssReplicationXLogCtl->insertHead; + writePtr = g_instance.xlog_cxt.ssReplicationXLogCtl->insertHead; + flushPtr = g_instance.xlog_cxt.ssReplicationXLogCtl->insertHead; } /* Get current timestamp. */ now = GetCurrentTimestamp(); @@ -2284,6 +2294,19 @@ Datum pg_stat_get_wal_receiver(PG_FUNCTION_ARGS) sndReplay = sendLocFix; } + if (SS_CLUSTER_DORADO_REPLICATION) { + ReadSSDoradoCtlInfoFile(); + XLogRecPtr sendLocFix = rcvReceived; + ShareStorageXLogCtl *ctlInfo = g_instance.xlog_cxt.ssReplicationXLogCtl; + if (XLByteLT(sendLocFix, ctlInfo->insertHead)) { + sendLocFix = ctlInfo->insertHead; + } + sndSent = sendLocFix; + sndWrite = sendLocFix; + sndFlush = sendLocFix; + sndReplay = sendLocFix; + } + SpinLockRelease(&walrcv->mutex); rc = memset_s(nulls, sizeof(nulls), 0, sizeof(nulls)); diff --git a/src/gausskernel/storage/replication/walsender.cpp b/src/gausskernel/storage/replication/walsender.cpp index 0da1ea0d9..2a26f6559 100755 --- a/src/gausskernel/storage/replication/walsender.cpp +++ b/src/gausskernel/storage/replication/walsender.cpp @@ -87,6 +87,7 @@ #include "replication/parallel_decode.h" #include "replication/parallel_decode_worker.h" #include "replication/parallel_reorderbuffer.h" +#include "replication/ss_cluster_replication.h" #include "storage/buf/bufmgr.h" #include "storage/smgr/fd.h" #include "storage/ipc.h" @@ -135,8 +136,8 @@ static int g_appname_extra_len = 3; /* [+]+\0 */ #define AmWalSenderToStandby() (t_thrd.walsender_cxt.MyWalSnd->sendRole == SNDROLE_PRIMARY_STANDBY) #define USE_PHYSICAL_XLOG_SEND \ - (AM_WAL_HADR_SENDER || !IS_SHARED_STORAGE_MODE || (walsnd->sendRole == SNDROLE_PRIMARY_BUILDSTANDBY)) -#define USE_SYNC_REP_FLUSH_PTR (AM_WAL_HADR_SENDER && !IS_SHARED_STORAGE_MODE) + (AM_WAL_HADR_SENDER || !SS_CLUSTER_DORADO_REPLICATION || !IS_SHARED_STORAGE_MODE || (walsnd->sendRole == SNDROLE_PRIMARY_BUILDSTANDBY)) +#define USE_SYNC_REP_FLUSH_PTR (AM_WAL_HADR_SENDER && (!IS_SHARED_STORAGE_MODE && !SS_CLUSTER_DORADO_REPLICATION)) /* Statistics for log control */ static const int MICROSECONDS_PER_SECONDS = 1000000; @@ -2920,8 +2921,15 @@ static void ProcessStandbyReplyMessage(void) /* * Advance our local xmin horizon when the client confirmed a flush. + * 1. When starting ss dorado replication, we need to know replayPtr that standby has already replayed, + * because primary xlog will cover standby xlog by Dorado synchronous replication. + * 2. Otherwise, we only need to confirm that standby xlog has been flushed successfully. */ - AdvanceReplicationSlot(reply.flush); + if (SS_CLUSTER_DORADO_REPLICATION) { + AdvanceReplicationSlot(reply.apply); + } else { + AdvanceReplicationSlot(reply.flush); + } if (AM_WAL_STANDBY_SENDER) { sndFlush = GetFlushRecPtr(); @@ -6266,6 +6274,15 @@ Datum pg_stat_get_wal_senders(PG_FUNCTION_ARGS) AlignFreeShareStorageCtl(ctlInfo); } + if (SS_CLUSTER_DORADO_REPLICATION && !AM_WAL_HADR_SENDER) { + ReadSSDoradoCtlInfoFile(); + ShareStorageXLogCtl *ctlInfo = g_instance.xlog_cxt.ssReplicationXLogCtl; + sentRecPtr = ctlInfo->insertHead; + sndWrite = ctlInfo->insertHead; + sndFlush = ctlInfo->insertHead; + sndReplay = ctlInfo->insertHead; + } + /* * if the walsener's pid has changed,we consider is is not a sync standby */ diff --git a/src/gausskernel/storage/smgr/segstore.cpp b/src/gausskernel/storage/smgr/segstore.cpp index 76c606bc5..7d5aae9e0 100755 --- a/src/gausskernel/storage/smgr/segstore.cpp +++ b/src/gausskernel/storage/smgr/segstore.cpp @@ -40,6 +40,7 @@ #include "storage/procarray.h" #include "ddes/dms/ss_transaction.h" #include "ddes/dms/ss_dms_bufmgr.h" +#include "replication/ss_cluster_replication.h" /* * This code manages relations that reside on segment-page storage. It implements functions used for smgr.cpp. * @@ -369,10 +370,8 @@ SegPageLocation seg_logic_to_physic_mapping(SMgrRelation reln, SegmentHead *seg_ BlockNumber blocknum; /* Recovery thread should use physical location to read data directly. */ - if (ENABLE_DMS && t_thrd.postmaster_cxt.HaShmData->current_mode == STANDBY_MODE && - g_instance.attr.attr_common.cluster_run_mode == RUN_MODE_STANDBY && - g_instance.attr.attr_storage.xlog_file_path != 0) { - ereport(DEBUG1, (errmsg("can segment address translation when role is SS_STANDBY_CLUSTER_MAIN_STANDBY"))); + if (SS_STANDBY_CLUSTER_MAIN_STANDBY || IS_SS_REPLICATION_MAIN_STANBY_NODE) { + ereport(DEBUG1, (errmsg("can segment address translation when role is IS_SS_REPLICATION_MAIN_STANBY_NODE"))); } else { if (RecoveryInProgress() && !CurrentThreadIsWorker() && !SS_IN_FLUSHCOPY) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("recovery is in progress"), diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h index 04fc92a38..f9eef7e32 100755 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@ -94,12 +94,12 @@ extern volatile uint64 sync_system_identifier; #define XLOG_FROM_STREAM (1 << 2) /* Streamed from master */ #define DORADO_STANDBY_CLUSTER (g_instance.attr.attr_common.cluster_run_mode == RUN_MODE_STANDBY && \ - g_instance.attr.attr_storage.xlog_file_path != 0) + g_instance.attr.attr_storage.xlog_file_path != NULL) #define DORADO_PRIMARY_CLUSTER (g_instance.attr.attr_common.cluster_run_mode == RUN_MODE_PRIMARY && \ - g_instance.attr.attr_storage.xlog_file_path != 0) + g_instance.attr.attr_storage.xlog_file_path != NULL) #define DORADO_STANDBY_CLUSTER_MAINSTANDBY_NODE ((t_thrd.postmaster_cxt.HaShmData->current_mode == STANDBY_MODE) && \ (g_instance.attr.attr_common.cluster_run_mode == RUN_MODE_STANDBY) && \ - (g_instance.attr.attr_storage.xlog_file_path != 0)) + (g_instance.attr.attr_storage.xlog_file_path != NULL)) /* * Recovery target type. diff --git a/src/include/access/xlog_basic.h b/src/include/access/xlog_basic.h index 18b5b9ed0..cb24df302 100644 --- a/src/include/access/xlog_basic.h +++ b/src/include/access/xlog_basic.h @@ -110,6 +110,8 @@ #define REDO_STATS_FILE "redo.state" #define REDO_STATS_FILE_TMP "redo.state.tmp" +#define SS_DORADO_CTRL_FILE (g_instance.datadir_cxt.controlInfoPath) + #define InvalidRepOriginId 0 #define InvalidXlogPreReadStartPtr 0xFFFFFFFFFFFFFFFF diff --git a/src/include/ddes/dms/ss_common_attr.h b/src/include/ddes/dms/ss_common_attr.h index 55abc990e..6ab77bfc0 100644 --- a/src/include/ddes/dms/ss_common_attr.h +++ b/src/include/ddes/dms/ss_common_attr.h @@ -101,51 +101,51 @@ (ENABLE_DMS && (t_thrd.xlog_cxt.server_mode == STANDBY_MODE || \ t_thrd.postmaster_cxt.HaShmData->current_mode == STANDBY_MODE) && \ (g_instance.attr.attr_common.cluster_run_mode == RUN_MODE_STANDBY) && \ - (g_instance.attr.attr_storage.xlog_file_path != 0)) + (g_instance.attr.attr_storage.xlog_file_path != NULL)) /* standby mode in primary or standby cluster */ #define SS_PRIMARY_STANDBY_CLUSTER_STANDBY \ (ENABLE_DMS && (t_thrd.xlog_cxt.server_mode == NORMAL_MODE || \ t_thrd.postmaster_cxt.HaShmData->current_mode == NORMAL_MODE) && \ - (g_instance.attr.attr_storage.xlog_file_path != 0)) + (g_instance.attr.attr_storage.xlog_file_path != NULL)) /* standby mode in primary cluster */ #define SS_PRIMARY_CLUSTER_STANDBY \ (ENABLE_DMS && (t_thrd.xlog_cxt.server_mode == NORMAL_MODE || \ t_thrd.postmaster_cxt.HaShmData->current_mode == NORMAL_MODE) && \ (g_instance.attr.attr_common.cluster_run_mode == RUN_MODE_PRIMARY) && \ - (g_instance.attr.attr_storage.xlog_file_path != 0)) + (g_instance.attr.attr_storage.xlog_file_path != NULL)) /* arbitrary mode when dorado hyperreplication and dms enabled */ #define SS_PRIMARY_STANDBY_CLUSTER_NORMAL \ (ENABLE_DMS && ((g_instance.attr.attr_common.cluster_run_mode == RUN_MODE_PRIMARY) || \ (g_instance.attr.attr_common.cluster_run_mode == RUN_MODE_STANDBY)) && \ - (g_instance.attr.attr_storage.xlog_file_path != 0)) + (g_instance.attr.attr_storage.xlog_file_path != NULL)) /* primary mode in primary cluster, after reform done and primary id has been determined */ #define SS_PRIMARY_CLUSTER_NORMAL_PRIMARY \ (SS_NORMAL_PRIMARY && (t_thrd.xlog_cxt.server_mode == PRIMARY_MODE || \ t_thrd.postmaster_cxt.HaShmData->current_mode == PRIMARY_MODE) && \ (g_instance.attr.attr_common.cluster_run_mode == RUN_MODE_PRIMARY) && \ - (g_instance.attr.attr_storage.xlog_file_path != 0)) + (g_instance.attr.attr_storage.xlog_file_path != NULL)) /* main standby in standby cluster, after reform done and primary id has been determined */ #define SS_STANDBY_CLUSTER_NORMAL_MAIN_STANDBY \ (SS_NORMAL_PRIMARY && (t_thrd.xlog_cxt.server_mode == STANDBY_MODE || \ t_thrd.postmaster_cxt.HaShmData->current_mode == STANDBY_MODE) && \ (g_instance.attr.attr_common.cluster_run_mode == RUN_MODE_STANDBY) && \ - (g_instance.attr.attr_storage.xlog_file_path != 0)) + (g_instance.attr.attr_storage.xlog_file_path != NULL)) /* standby mode in standby cluster, after reform done and primary id has been determined */ #define SS_STANDBY_CLUSTER_NORMAL_STANDBY \ (SS_NORMAL_STANDBY && (t_thrd.xlog_cxt.server_mode == STANDBY_MODE || \ t_thrd.postmaster_cxt.HaShmData->current_mode == STANDBY_MODE) && \ (g_instance.attr.attr_common.cluster_run_mode == RUN_MODE_STANDBY) && \ - (g_instance.attr.attr_storage.xlog_file_path != 0)) + (g_instance.attr.attr_storage.xlog_file_path != NULL)) /* standby mode in primary or standby, after reform done and primary id has been determined */ #define SS_PRIMARY_STANDBY_CLUSTER_NORMAL_STANDBY \ - (SS_NORMAL_STANDBY && (g_instance.attr.attr_storage.xlog_file_path != 0)) + (SS_NORMAL_STANDBY && (g_instance.attr.attr_storage.xlog_file_path != NULL)) #define SS_CLUSTER_ONDEMAND_NOT_NORAML \ (ENABLE_DMS && (g_instance.dms_cxt.SSReformerControl.clusterStatus != CLUSTER_NORMAL)) diff --git a/src/include/knl/knl_guc/knl_instance_attr_storage.h b/src/include/knl/knl_guc/knl_instance_attr_storage.h index 8d17c7e67..f68282360 100755 --- a/src/include/knl/knl_guc/knl_instance_attr_storage.h +++ b/src/include/knl/knl_guc/knl_instance_attr_storage.h @@ -212,6 +212,7 @@ typedef struct knl_instance_attr_storage { #endif bool enable_huge_pages; int huge_page_size; + bool enable_ss_dorado; } knl_instance_attr_storage; #endif /* SRC_INCLUDE_KNL_KNL_INSTANCE_ATTR_STORAGE_H_ */ diff --git a/src/include/knl/knl_instance.h b/src/include/knl/knl_instance.h index 461b1e36c..e481d5e52 100755 --- a/src/include/knl/knl_instance.h +++ b/src/include/knl/knl_instance.h @@ -900,6 +900,7 @@ typedef struct knl_g_xlog_context { void *shareStorageXLogCtlOrigin; ShareStorageOperateCtl shareStorageopCtl; int shareStorageLockFd; + ShareStorageXLogCtl *ssReplicationXLogCtl; } knl_g_xlog_context; typedef struct knl_g_undo_context { @@ -1195,6 +1196,7 @@ typedef struct knl_g_datadir_context { char xlogDir[MAXPGPATH]; char controlPath[MAXPGPATH]; char controlBakPath[MAXPGPATH]; + char controlInfoPath[MAXPGPATH]; knl_g_dwsubdatadir_context dw_subdir_cxt; } knl_g_datadir_context; diff --git a/src/include/replication/shared_storage_walreceiver.h b/src/include/replication/shared_storage_walreceiver.h index 0ccf59bda..39d82b1e6 100644 --- a/src/include/replication/shared_storage_walreceiver.h +++ b/src/include/replication/shared_storage_walreceiver.h @@ -40,31 +40,33 @@ extern bool SharedStorageXlogReadCheck(XLogReaderState *xlogreader, XLogRecPtr r #define IS_SHARED_STORAGE_MAIN_STANDBY_MODE \ (t_thrd.postmaster_cxt.HaShmData->is_hadr_main_standby && \ - g_instance.attr.attr_storage.xlog_file_path != 0) + g_instance.attr.attr_storage.xlog_file_path != NULL) #define IS_SHARED_STORAGE_CASCADE_STANDBY_MODE \ (t_thrd.postmaster_cxt.HaShmData->is_cascade_standby && \ - g_instance.attr.attr_storage.xlog_file_path != 0) + g_instance.attr.attr_storage.xlog_file_path != NULL) #define IS_SHARED_STORAGE_STANDBY_CLUSTER \ (g_instance.attr.attr_common.cluster_run_mode == RUN_MODE_STANDBY && \ - g_instance.attr.attr_storage.xlog_file_path != 0) + g_instance.attr.attr_storage.xlog_file_path != NULL) #define IS_SHARED_STORAGE_STANDBY_CLUSTER_STANDBY_MODE \ (t_thrd.xlog_cxt.server_mode == STANDBY_MODE && \ g_instance.attr.attr_common.cluster_run_mode == RUN_MODE_STANDBY && \ - g_instance.attr.attr_storage.xlog_file_path != 0) + g_instance.attr.attr_storage.xlog_file_path != NULL) #define IS_SHARED_STORAGE_PRIMARY_CLUSTER_STANDBY_MODE \ (t_thrd.xlog_cxt.server_mode == STANDBY_MODE && \ g_instance.attr.attr_common.cluster_run_mode == RUN_MODE_PRIMARY && \ - g_instance.attr.attr_storage.xlog_file_path != 0) + g_instance.attr.attr_storage.xlog_file_path != NULL) #define IS_SHARED_STORAGE_PRIMARY_CLUSTER_PRIMARY_MODE \ (((IS_PGXC_DATANODE && t_thrd.xlog_cxt.server_mode == PRIMARY_MODE) || \ (IS_PGXC_COORDINATOR && t_thrd.xlog_cxt.server_mode == NORMAL_MODE)) && \ g_instance.attr.attr_common.cluster_run_mode == RUN_MODE_PRIMARY && \ - g_instance.attr.attr_storage.xlog_file_path != 0) + g_instance.attr.attr_storage.xlog_file_path != NULL) #define IS_SHARED_STORAGE_STANBY_MODE \ - (g_instance.attr.attr_storage.xlog_file_path != 0 && t_thrd.xlog_cxt.server_mode == STANDBY_MODE) -#define IS_SHARED_STORAGE_MODE (g_instance.attr.attr_storage.xlog_file_path != 0) + (g_instance.attr.attr_storage.xlog_file_path != NULL && t_thrd.xlog_cxt.server_mode == STANDBY_MODE) +#define IS_SHARED_STORAGE_MODE (g_instance.attr.attr_storage.xlog_file_path != NULL) + +#define MAX_XLOG_FILE_SIZE_BUFFER 256 #endif /* INCLUDE_REPLICATION_SHARED_STORAGE_WALRECEIVER_H_ */ diff --git a/src/include/replication/ss_cluster_replication.h b/src/include/replication/ss_cluster_replication.h new file mode 100644 index 000000000..70a0b4c21 --- /dev/null +++ b/src/include/replication/ss_cluster_replication.h @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * Portions Copyright (c) 2021, openGauss Contributors + * + * Description: openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * --------------------------------------------------------------------------------------- + * + * + * + * IDENTIFICATION + * src/include/replication/ss_cluster_replication.h + * + * --------------------------------------------------------------------------------------- + */ + +#ifndef INCLUDE_REPLICATION_SS_CLUSTER_REPLICATION_H_ +#define INCLUDE_REPLICATION_SS_CLUSTER_REPLICATION_H_ + +#include "postgres.h" +#include "replication/walprotocol.h" +#include "knl/knl_instance.h" +#include + +const uint32 SS_DORADO_CTL_INFO_SIZE = 512; + +#define SS_CLUSTER_DORADO_REPLICATION \ + (ENABLE_DSS && g_instance.attr.attr_storage.enable_ss_dorado) + +/* Primary Cluster in SS replication */ +#define SS_REPLICATION_MAIN_CLUSTER \ + (SS_CLUSTER_DORADO_REPLICATION && (g_instance.attr.attr_common.cluster_run_mode == RUN_MODE_PRIMARY)) + +/* Standby Cluster in SS replication */ +#define SS_REPLICATION_STANDBY_CLUSTER \ + (SS_CLUSTER_DORADO_REPLICATION && (g_instance.attr.attr_common.cluster_run_mode == RUN_MODE_STANDBY)) + +/* Primary node in SS replication, means primary node in main cluster. */ +#define IS_SS_REPLICATION_PRIMARY_NODE \ + (SS_REPLICATION_MAIN_CLUSTER && (t_thrd.postmaster_cxt.HaShmData->current_mode == PRIMARY_MODE)) + +/* Standby node in SS replication, means standby node in main cluster. */ +#define IS_SS_REPLICATION_PRIMARY_CLUSTER_STANDBY_NODE \ + (SS_REPLICATION_MAIN_CLUSTER && (t_thrd.postmaster_cxt.HaShmData->current_mode == NORMAL_MODE)) + +/* Main standby node in SS replication, means primary node in standby cluster. */ +#define IS_SS_REPLICATION_MAIN_STANBY_NODE \ + (SS_REPLICATION_STANDBY_CLUSTER && (t_thrd.postmaster_cxt.HaShmData->current_mode == STANDBY_MODE)) + +/* Standby node in SS replication, means standby node in standby cluster. */ +#define IS_SS_REPLICATION_STANDBY_CLUSTER_STANDBY_NODE \ + (SS_REPLICATION_STANDBY_CLUSTER && (t_thrd.postmaster_cxt.HaShmData->current_mode == NORMAL_MODE)) + +/* All Standby in SS replication, means nodes other than primary node in primary cluster and standby cluster */ +#define IS_SS_REPLICATION_STANBY_NODE \ + (SS_CLUSTER_DORADO_REPLICATION && (t_thrd.postmaster_cxt.HaShmData->current_mode == NORMAL_MODE)) + + +void SSClusterDoradoStorageInit(); +void InitSSDoradoCtlInfo(ShareStorageXLogCtl *ctlInfo, uint64 sysidentifier); +void UpdateSSDoradoCtlInfoAndSync(); +void WriteSSDoradoCtlInfoFile(); +void ReadSSDoradoCtlInfoFile(); +void CheckSSDoradoCtlInfo(XLogRecPtr localEnd); +#endif // INCLUDE_REPLICATION_SS_CLUSTER_REPLICATION_H_ \ No newline at end of file diff --git a/src/include/storage/dorado_operation/dorado_fd.h b/src/include/storage/dorado_operation/dorado_fd.h index 5d9dc86c5..de25e39e4 100644 --- a/src/include/storage/dorado_operation/dorado_fd.h +++ b/src/include/storage/dorado_operation/dorado_fd.h @@ -30,6 +30,7 @@ #include "knl/knl_variable.h" const uint32 MEMORY_ALIGNED_SIZE = 8192; +const uint32 DORADO_CTL_WRITE_SIZE = 512; void InitDoradoStorage(char *filePath, uint64 fileSize); diff --git a/src/include/tool_common.h b/src/include/tool_common.h index e126f0a90..b6a3bd36d 100644 --- a/src/include/tool_common.h +++ b/src/include/tool_common.h @@ -102,6 +102,7 @@ typedef struct st_datadir_t { char multixactDir[MAXPGPATH]; char controlPath[MAXPGPATH]; char controlBakPath[MAXPGPATH]; + char controlInfoPath[MAXPGPATH]; dw_subdatadir_t dwDir; } datadir_t; diff --git a/src/test/regress/output/recovery_2pc_tools.source b/src/test/regress/output/recovery_2pc_tools.source index bd3ed7efc..7d5db9acb 100644 --- a/src/test/regress/output/recovery_2pc_tools.source +++ b/src/test/regress/output/recovery_2pc_tools.source @@ -325,6 +325,7 @@ select name,vartype,unit,min_val,max_val from pg_settings where name <> 'qunit_c enable_sonic_optspill | bool | | | enable_sort | bool | | | enable_sortgroup_agg | bool | | | + enable_ss_dorado | bool | | | enable_startwith_debug | bool | | | enable_stmt_track | bool | | | enable_stream_replication | bool | | | From 7ecd24c7177302127cf766a89d7117676998ac5d Mon Sep 17 00:00:00 2001 From: zhangchao Date: Thu, 10 Aug 2023 14:22:30 +0800 Subject: [PATCH 117/304] =?UTF-8?q?pg=5Fregress.cpp=E5=86=85=E5=AD=98?= =?UTF-8?q?=E6=B3=84=E9=9C=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/test/regress/pg_regress.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/regress/pg_regress.cpp b/src/test/regress/pg_regress.cpp index 090b937ce..34b0c7e26 100644 --- a/src/test/regress/pg_regress.cpp +++ b/src/test/regress/pg_regress.cpp @@ -1991,7 +1991,7 @@ static void initdb_node_info(bool standby) char buf[MAXPGPATH * 4]; char* data_folder = get_node_info_name(i, DATANODE, false); - char* node_name = get_node_info_name(i, DATANODE, true; + char* node_name = get_node_info_name(i, DATANODE, true); (void)snprintf(buf, sizeof(buf), SYSTEMQUOTE "\"%s/gs_initdb\" --nodename %s %s -w \"gauss@123\" -D \"%s/%s_standby\" -L \"%s\" " From 6ba7e7584696e5c256fcc783cc0215eb2cf34594 Mon Sep 17 00:00:00 2001 From: "arcoalien@qq.com" Date: Thu, 3 Aug 2023 17:25:12 +0800 Subject: [PATCH 118/304] =?UTF-8?q?=E8=B5=84=E6=BA=90=E6=B1=A0=E5=8C=96?= =?UTF-8?q?=E6=94=AF=E6=8C=81=E4=B8=BB=E6=9C=BA=E5=B9=BF=E6=92=AD=E6=9C=80?= =?UTF-8?q?=E6=96=B0=E5=BF=AB=E7=85=A7=E7=BB=99=E5=A4=87=E6=9C=BA=EF=BC=8C?= =?UTF-8?q?=E5=A4=87=E6=9C=BA=E6=94=AF=E6=8C=81=E7=BC=93=E5=AD=98=E5=85=A8?= =?UTF-8?q?=E5=B1=80=E5=BF=AB=E7=85=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/gs_guc/cluster_guc.conf | 1 + .../backend/utils/misc/guc/guc_storage.cpp | 11 ++ .../ddes/adapter/ss_dms_callback.cpp | 3 + .../ddes/adapter/ss_transaction.cpp | 102 +++++++++++++++++- .../process/threadpool/knl_instance.cpp | 5 +- .../process/threadpool/knl_thread.cpp | 3 + src/gausskernel/storage/ipc/procarray.cpp | 6 ++ src/include/ddes/dms/ss_common_attr.h | 3 + src/include/ddes/dms/ss_transaction.h | 9 ++ .../knl/knl_guc/knl_instance_attr_storage.h | 1 + src/include/knl/knl_instance.h | 4 + src/include/knl/knl_thread.h | 3 + .../regress/output/recovery_2pc_tools.source | 1 + 13 files changed, 146 insertions(+), 6 deletions(-) diff --git a/src/bin/gs_guc/cluster_guc.conf b/src/bin/gs_guc/cluster_guc.conf index c59516547..9c1b45024 100755 --- a/src/bin/gs_guc/cluster_guc.conf +++ b/src/bin/gs_guc/cluster_guc.conf @@ -751,6 +751,7 @@ enable_remote_excute|bool|0,0|NULL|NULL| light_comm|bool|0,0|NULL|NULL| ignore_standby_lsn_window|int|0,2147483647|ms|NULL| ignore_feedback_xmin_window|int|0,2147483647|ms|NULL| +ss_enable_bcast_snapshot|bool|0,0|NULL|NULL| [cmserver] log_dir|string|0,0|NULL|NULL| log_file_size|int|0,2047|MB|NULL| diff --git a/src/common/backend/utils/misc/guc/guc_storage.cpp b/src/common/backend/utils/misc/guc/guc_storage.cpp index 7306eca39..0ec2764b3 100755 --- a/src/common/backend/utils/misc/guc/guc_storage.cpp +++ b/src/common/backend/utils/misc/guc/guc_storage.cpp @@ -1185,6 +1185,17 @@ static void InitStorageConfigureNamesBool() NULL}, #endif + {{"ss_enable_bcast_snapshot", + PGC_POSTMASTER, + NODE_SINGLENODE, + SHARED_STORAGE_OPTIONS, + gettext_noop("Enable broadcast snapshot by primay."), + NULL}, + &g_instance.attr.attr_storage.dms_attr.enable_bcast_snapshot, + false, + NULL, + NULL, + NULL}, {{"enable_huge_pages", PGC_POSTMASTER, NODE_SINGLENODE, diff --git a/src/gausskernel/ddes/adapter/ss_dms_callback.cpp b/src/gausskernel/ddes/adapter/ss_dms_callback.cpp index 6ebada131..bbe8262c2 100644 --- a/src/gausskernel/ddes/adapter/ss_dms_callback.cpp +++ b/src/gausskernel/ddes/adapter/ss_dms_callback.cpp @@ -1058,6 +1058,9 @@ static int32 CBProcessBroadcast(void *db_handle, char *data, unsigned int len, c case BCAST_CHECK_DB_BACKENDS: ret = SSCheckDbBackends(data, len, output_msg, output_msg_len); break; + case BCAST_SEND_SNAPSHOT: + ret = SSUpdateLatestSnapshotOfStandby(data, len); + break; default: ereport(WARNING, (errmodule(MOD_DMS), errmsg("invalid broadcast operate type"))); ret = DMS_ERROR; diff --git a/src/gausskernel/ddes/adapter/ss_transaction.cpp b/src/gausskernel/ddes/adapter/ss_transaction.cpp index 77e2ed168..d03c2d481 100644 --- a/src/gausskernel/ddes/adapter/ss_transaction.cpp +++ b/src/gausskernel/ddes/adapter/ss_transaction.cpp @@ -49,15 +49,11 @@ static inline void txnstatusHashStats(uint64 timeDiff); void SSStandbyGlobalInvalidSharedInvalidMessages(const SharedInvalidationMessage* msg, Oid tsid); -Snapshot SSGetSnapshotData(Snapshot snapshot) +static Snapshot SSGetSnapshotDataFromMaster(Snapshot snapshot) { dms_opengauss_txn_snapshot_t dms_snapshot; dms_context_t dms_ctx; InitDmsContext(&dms_ctx); - if (SS_IN_REFORM) { - ereport(DEBUG1, (errmsg("[SS reform] SSGetSnapshotData returns NULL in reform."))); - return NULL; - } do { dms_ctx.xmap_ctx.dest_id = (unsigned int)SS_PRIMARY_ID; @@ -76,6 +72,57 @@ Snapshot SSGetSnapshotData(Snapshot snapshot) snapshot->xmin = dms_snapshot.xmin; snapshot->xmax = dms_snapshot.xmax; snapshot->snapshotcsn = dms_snapshot.snapshotcsn; + if (ENABLE_SS_BCAST_SNAPSHOT) { + t_thrd.dms_cxt.latest_snapshot_xmin = dms_snapshot.xmin; + t_thrd.dms_cxt.latest_snapshot_csn = dms_snapshot.snapshotcsn; + t_thrd.dms_cxt.latest_snapshot_xmax = dms_snapshot.xmax; + ereport(DEBUG1, + (errmsg("Get Local snapshot from master, xmin/xmax/csn:%lu/%lu/%lu", t_thrd.dms_cxt.latest_snapshot_xmin, + t_thrd.dms_cxt.latest_snapshot_xmax, t_thrd.dms_cxt.latest_snapshot_csn))); + } + if (!TransactionIdIsValid(t_thrd.pgxact->xmin)) { + t_thrd.pgxact->xmin = u_sess->utils_cxt.TransactionXmin = snapshot->xmin; + } + + if (!TransactionIdIsNormal(u_sess->utils_cxt.RecentGlobalXmin)) { + u_sess->utils_cxt.RecentGlobalXmin = FirstNormalTransactionId; + } + u_sess->utils_cxt.RecentGlobalDataXmin = u_sess->utils_cxt.RecentGlobalXmin; + u_sess->utils_cxt.RecentXmin = snapshot->xmin; + return snapshot; +} + +Snapshot SSGetSnapshotData(Snapshot snapshot) +{ + if (SS_IN_REFORM) { + ereport(DEBUG1, (errmsg("[SS reform] SSGetSnapshotData returns NULL in reform."))); + return NULL; + } + + if (!ENABLE_SS_BCAST_SNAPSHOT || + (g_instance.dms_cxt.latest_snapshot_xmax == InvalidTransactionId && + t_thrd.dms_cxt.latest_snapshot_xmax == InvalidTransactionId)) { + return SSGetSnapshotDataFromMaster(snapshot); + } else if (TransactionIdPrecedes(t_thrd.dms_cxt.latest_snapshot_xmax, g_instance.dms_cxt.latest_snapshot_xmax)) { + SpinLockAcquire(&g_instance.dms_cxt.set_snapshot_mutex); + t_thrd.dms_cxt.latest_snapshot_xmin = g_instance.dms_cxt.latest_snapshot_xmin; + t_thrd.dms_cxt.latest_snapshot_csn = g_instance.dms_cxt.latest_snapshot_csn; + t_thrd.dms_cxt.latest_snapshot_xmax = g_instance.dms_cxt.latest_snapshot_xmax; + SpinLockRelease(&g_instance.dms_cxt.set_snapshot_mutex); + ereport(DEBUG1, + (errmsg("Update Local snapshot from global, xmin/xmax/csn:%lu/%lu/%lu", t_thrd.dms_cxt.latest_snapshot_xmin, + t_thrd.dms_cxt.latest_snapshot_xmax, t_thrd.dms_cxt.latest_snapshot_csn))); + } + + Assert(t_thrd.dms_cxt.latest_snapshot_xmax != InvalidTransactionId); + if (g_instance.dms_cxt.latest_snapshot_xmax == InvalidTransactionId) { + return SSGetSnapshotDataFromMaster(snapshot); + } + snapshot->xmin = t_thrd.dms_cxt.latest_snapshot_xmin; + snapshot->xmax = t_thrd.dms_cxt.latest_snapshot_xmax; + snapshot->snapshotcsn = t_thrd.dms_cxt.latest_snapshot_csn; + ereport(DEBUG1, (errmsg("Current Use snapshot, xmin/xmax/csn:%lu/%lu/%lu", snapshot->xmin, + snapshot->xmax, snapshot->snapshotcsn))); if (!TransactionIdIsValid(t_thrd.pgxact->xmin)) { t_thrd.pgxact->xmin = u_sess->utils_cxt.TransactionXmin = snapshot->xmin; } @@ -88,6 +135,29 @@ Snapshot SSGetSnapshotData(Snapshot snapshot) return snapshot; } +int SSUpdateLatestSnapshotOfStandby(char *data, uint32 len) +{ + if (unlikely(len != sizeof(SSBroadcastSnapshot))) { + ereport(DEBUG1, (errmsg("invalid broadcast set snapshot message"))); + return DMS_ERROR; + } + + SSBroadcastSnapshot *received_data = (SSBroadcastSnapshot *)data; + if (TransactionIdPrecedes(received_data->xmax, g_instance.dms_cxt.latest_snapshot_xmax)) { + ereport(WARNING, (errmsg("Receive oldest one, can't update:%lu/%lu", received_data->xmin, + g_instance.dms_cxt.latest_snapshot_xmax))); + return DMS_SUCCESS; + } + SpinLockAcquire(&g_instance.dms_cxt.set_snapshot_mutex); + g_instance.dms_cxt.latest_snapshot_xmin = received_data->xmin; + g_instance.dms_cxt.latest_snapshot_csn = received_data->csn; + g_instance.dms_cxt.latest_snapshot_xmax = received_data->xmax; + SpinLockRelease(&g_instance.dms_cxt.set_snapshot_mutex); + ereport(DEBUG1, (errmsg("Receive and set global snapshot xmin/xmax/csn:%lu/%lu/%lu", received_data->xmin, + received_data->xmax, received_data->csn))); + return DMS_SUCCESS; +} + static int SSTransactionIdGetCSN(dms_opengauss_xid_csn_t *dms_txn_info, dms_opengauss_csn_result_t *xid_csn_result) { dms_context_t dms_ctx; @@ -384,6 +454,28 @@ TransactionId SSMultiXactIdGetUpdateXid(TransactionId xmax, uint16 t_infomask, u return update_xid; } +void SSSendLatestSnapshotToStandby(TransactionId xmin, TransactionId xmax, CommitSeqNo csn) +{ + dms_context_t dms_ctx; + InitDmsContext(&dms_ctx); + int ret; + SSBroadcastSnapshot latest_snapshot; + latest_snapshot.xmin = xmin; + latest_snapshot.xmax = xmax; + latest_snapshot.csn = csn; + latest_snapshot.type = BCAST_SEND_SNAPSHOT; + do { + ret = dms_broadcast_msg(&dms_ctx, (char *)&latest_snapshot, sizeof(SSBroadcastSnapshot), + (unsigned char)false, SS_BROADCAST_WAIT_ONE_SECOND); + + if (ret == DMS_SUCCESS) { + return; + } + + pg_usleep(5000L); + } while (ret != DMS_SUCCESS); +} + void SSIsPageHitDms(RelFileNode& node, BlockNumber page, int pagesNum, uint64 *pageMap, int *bitCount) { dms_context_t dms_ctx; diff --git a/src/gausskernel/process/threadpool/knl_instance.cpp b/src/gausskernel/process/threadpool/knl_instance.cpp index 09b54dd36..5439c8679 100755 --- a/src/gausskernel/process/threadpool/knl_instance.cpp +++ b/src/gausskernel/process/threadpool/knl_instance.cpp @@ -202,7 +202,6 @@ static void knl_g_dms_init(knl_g_dms_context *dms_cxt) dms_cxt->resetSyscache = false; dms_cxt->finishedRecoverOldPrimaryDWFile = false; dms_cxt->dw_init = false; - { ss_xmin_info_t *xmin_info = &g_instance.dms_cxt.SSXminInfo; for (int i = 0; i < DMS_MAX_INSTANCES; i++) { @@ -220,6 +219,10 @@ static void knl_g_dms_init(knl_g_dms_context *dms_cxt) SpinLockInit(&xmin_info->bitmap_active_nodes_lock); xmin_info->bitmap_active_nodes = 0; } + dms_cxt->latest_snapshot_xmin = 0; + dms_cxt->latest_snapshot_xmax = 0; + dms_cxt->latest_snapshot_csn = 0; + SpinLockInit(&dms_cxt->set_snapshot_mutex); } static void knl_g_tests_init(knl_g_tests_context* tests_cxt) diff --git a/src/gausskernel/process/threadpool/knl_thread.cpp b/src/gausskernel/process/threadpool/knl_thread.cpp index 6bae653ad..c321ac6e1 100755 --- a/src/gausskernel/process/threadpool/knl_thread.cpp +++ b/src/gausskernel/process/threadpool/knl_thread.cpp @@ -1711,6 +1711,9 @@ static void knl_t_dms_context_init(knl_t_dms_context *dms_cxt) dms_cxt->flush_copy_get_page_failed = false; dms_cxt->SSTxnStatusHash = NULL; dms_cxt->SSTxnStatusLRU = NULL; + dms_cxt->latest_snapshot_xmin = 0; + dms_cxt->latest_snapshot_xmax = 0; + dms_cxt->latest_snapshot_csn = 0; } static void knl_t_ondemand_xlog_copy_context_init(knl_t_ondemand_xlog_copy_context *ondemand_xlog_copy_cxt) diff --git a/src/gausskernel/storage/ipc/procarray.cpp b/src/gausskernel/storage/ipc/procarray.cpp index 7511bcac8..19738962b 100755 --- a/src/gausskernel/storage/ipc/procarray.cpp +++ b/src/gausskernel/storage/ipc/procarray.cpp @@ -5030,6 +5030,9 @@ void CalculateLocalLatestSnapshot(bool forceCalc) if (ENABLE_DMS && SS_PRIMARY_MODE) { SSUpdateNodeOldestXmin(SS_MY_INST_ID, globalxmin); globalxmin = SSGetGlobalOldestXmin(globalxmin); + if (ENABLE_SS_BCAST_SNAPSHOT) { + SSSendLatestSnapshotToStandby(xmin, xmax, t_thrd.xact_cxt.ShmemVariableCache->nextCommitSeqNo); + } } t_thrd.xact_cxt.ShmemVariableCache->xmin = xmin; @@ -5037,6 +5040,9 @@ void CalculateLocalLatestSnapshot(bool forceCalc) if (GTM_FREE_MODE) { t_thrd.xact_cxt.ShmemVariableCache->recentGlobalXmin = globalxmin; } + } else if (ENABLE_SS_BCAST_SNAPSHOT && SS_PRIMARY_MODE) { + SSSendLatestSnapshotToStandby(t_thrd.xact_cxt.ShmemVariableCache->xmin, xmax, + t_thrd.xact_cxt.ShmemVariableCache->nextCommitSeqNo); } if (GTM_LITE_MODE) { diff --git a/src/include/ddes/dms/ss_common_attr.h b/src/include/ddes/dms/ss_common_attr.h index 55abc990e..eb039e38b 100644 --- a/src/include/ddes/dms/ss_common_attr.h +++ b/src/include/ddes/dms/ss_common_attr.h @@ -36,11 +36,13 @@ #define ENABLE_REFORM false #define ENABLE_VERIFY_PAGE_VERSION false #define ENABLE_SS_TXNSTATUS_CACHE false +#define ENABLE_SS_BCAST_SNAPSHOT false #else #define ENABLE_DMS (g_instance.attr.attr_storage.dms_attr.enable_dms && !IsInitdb) #define ENABLE_REFORM (g_instance.attr.attr_storage.dms_attr.enable_reform) #define ENABLE_VERIFY_PAGE_VERSION (g_instance.attr.attr_storage.dms_attr.enable_verify_page) #define ENABLE_SS_TXNSTATUS_CACHE (ENABLE_DMS && g_instance.attr.attr_storage.dms_attr.txnstatus_cache_size > 0) +#define ENABLE_SS_BCAST_SNAPSHOT (ENABLE_DMS && g_instance.attr.attr_storage.dms_attr.enable_bcast_snapshot) #endif #define SS_REFORM_REFORMER \ @@ -202,6 +204,7 @@ typedef enum SSBroadcastOp { BCAST_DDLLOCKRELEASE, BCAST_DDLLOCKRELEASE_ALL, BCAST_CHECK_DB_BACKENDS, + BCAST_SEND_SNAPSHOT, BCAST_END } SSBroadcastOp; diff --git a/src/include/ddes/dms/ss_transaction.h b/src/include/ddes/dms/ss_transaction.h index 721c38bb8..723da9f92 100644 --- a/src/include/ddes/dms/ss_transaction.h +++ b/src/include/ddes/dms/ss_transaction.h @@ -42,6 +42,13 @@ typedef struct SSBroadcastXminAck { TransactionId xmin; } SSBroadcastXminAck; +typedef struct SSBroadcastSnapshot { + SSBroadcastOp type; // must be first + TransactionId xmin; + TransactionId xmax; + CommitSeqNo csn; +} SSBroadcastSnapshot; + typedef struct SSBroadcastSI { SSBroadcastOp type; // must be first Oid tablespaceid; @@ -113,5 +120,7 @@ int SSCheckDbBackends(char *data, uint32 len, char *output_msg, uint32 *output_m int SSCheckDbBackendsAck(char *data, unsigned int len); bool SSCheckDbBackendsFromAllStandby(Oid dbid); void SSStandbyUpdateRedirectInfo(); +void SSSendLatestSnapshotToStandby(TransactionId xmin, TransactionId xmax, CommitSeqNo csn); +int SSUpdateLatestSnapshotOfStandby(char *data, uint32 len); #endif diff --git a/src/include/knl/knl_guc/knl_instance_attr_storage.h b/src/include/knl/knl_guc/knl_instance_attr_storage.h index 8d17c7e67..f0be1a274 100755 --- a/src/include/knl/knl_guc/knl_instance_attr_storage.h +++ b/src/include/knl/knl_guc/knl_instance_attr_storage.h @@ -125,6 +125,7 @@ typedef struct knl_instance_attr_dms { int32 sslog_max_file_size; //Unit:KB int parallel_thread_num; int32 txnstatus_cache_size; + bool enable_bcast_snapshot; } knl_instance_attr_dms; typedef struct knl_instance_attr_storage { diff --git a/src/include/knl/knl_instance.h b/src/include/knl/knl_instance.h index 461b1e36c..ccbbf5a98 100755 --- a/src/include/knl/knl_instance.h +++ b/src/include/knl/knl_instance.h @@ -1213,6 +1213,10 @@ typedef struct knl_g_dms_context { bool resetSyscache; bool finishedRecoverOldPrimaryDWFile; bool dw_init; + uint64 latest_snapshot_xmin; + uint64 latest_snapshot_xmax; + uint64 latest_snapshot_csn; + slock_t set_snapshot_mutex; char dmsInstAddr[MAX_REPLNODE_NUM][DMS_MAX_IP_LEN]; char conninfo[MAXPGPATH]; ss_dfx_stats_t SSDFxStats; diff --git a/src/include/knl/knl_thread.h b/src/include/knl/knl_thread.h index a56f99cc9..06cd5f6ce 100755 --- a/src/include/knl/knl_thread.h +++ b/src/include/knl/knl_thread.h @@ -3350,6 +3350,9 @@ typedef struct knl_t_dms_context { bool buf_in_aio; bool is_reform_proc; bool CloseAllSessionsFailed; + uint64 latest_snapshot_xmin; + uint64 latest_snapshot_xmax; + uint64 latest_snapshot_csn; char *origin_buf; /* origin buffer for unaligned read/write */ char *aligned_buf; int size; /* aligned buffer size */ diff --git a/src/test/regress/output/recovery_2pc_tools.source b/src/test/regress/output/recovery_2pc_tools.source index bd3ed7efc..6b9493cbd 100644 --- a/src/test/regress/output/recovery_2pc_tools.source +++ b/src/test/regress/output/recovery_2pc_tools.source @@ -620,6 +620,7 @@ select name,vartype,unit,min_val,max_val from pg_settings where name <> 'qunit_c ss_dss_conn_path | string | | | ss_dss_vg_name | string | | | ss_enable_aio | bool | | | + ss_enable_bcast_snapshot | bool | | | ss_enable_catalog_centralized | bool | | | ss_enable_dms | bool | | | ss_enable_dss | bool | | | From 560d961b346c5bf2fa25276e08aba6e5e5978bd1 Mon Sep 17 00:00:00 2001 From: duzhuolin Date: Thu, 10 Aug 2023 14:15:06 +0000 Subject: [PATCH 119/304] update src/common/backend/catalog/index.cpp. Signed-off-by: duzhuolin --- src/common/backend/catalog/index.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/common/backend/catalog/index.cpp b/src/common/backend/catalog/index.cpp index 3cf78ce9c..34397d322 100644 --- a/src/common/backend/catalog/index.cpp +++ b/src/common/backend/catalog/index.cpp @@ -4422,10 +4422,8 @@ double* GetGlobalIndexTuplesForSubPartition(Relation heapRelation, Relation inde partition = partitionOpen(heapRelation, partitionId, ShareLock); heapPartRel = partitionGetRelation(heapRelation, partition); subPartitionIdList = relationGetPartitionOidList(heapPartRel); - int subPartNum = 0; - if (subPartitionIdList != NULL) { - subPartNum = subPartitionIdList->length; - } + int subPartNum = subPartitionIdList->length; + Assert(subPartitionIdList != NULL); if (globalIndexTuples != NULL) { globalIndexTuples = (double*)repalloc(globalIndexTuples, (subPartitionIdx + subPartNum) * sizeof(double)); } else { From 9b98e9d5a42446cb8aab6fd25bc0f7f7a6f8033e Mon Sep 17 00:00:00 2001 From: duzhuolin Date: Fri, 11 Aug 2023 01:46:55 +0000 Subject: [PATCH 120/304] update src/common/backend/catalog/index.cpp. Signed-off-by: duzhuolin --- src/common/backend/catalog/index.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/common/backend/catalog/index.cpp b/src/common/backend/catalog/index.cpp index 34397d322..e5a9baf95 100644 --- a/src/common/backend/catalog/index.cpp +++ b/src/common/backend/catalog/index.cpp @@ -4422,8 +4422,9 @@ double* GetGlobalIndexTuplesForSubPartition(Relation heapRelation, Relation inde partition = partitionOpen(heapRelation, partitionId, ShareLock); heapPartRel = partitionGetRelation(heapRelation, partition); subPartitionIdList = relationGetPartitionOidList(heapPartRel); - int subPartNum = subPartitionIdList->length; Assert(subPartitionIdList != NULL); + int subPartNum = subPartitionIdList->length; + if (globalIndexTuples != NULL) { globalIndexTuples = (double*)repalloc(globalIndexTuples, (subPartitionIdx + subPartNum) * sizeof(double)); } else { From df0b3b7f61aaf485cc85c0aa68e6c444d13fa5b2 Mon Sep 17 00:00:00 2001 From: yaojun Date: Fri, 11 Aug 2023 10:16:25 +0800 Subject: [PATCH 121/304] fix typo error: CanTupleInertGSC--->CanTupleInsertGSC --- src/common/backend/utils/cache/knl_globalsystupcache.cpp | 8 ++++---- src/include/utils/knl_globalsyscache_common.h | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/common/backend/utils/cache/knl_globalsystupcache.cpp b/src/common/backend/utils/cache/knl_globalsystupcache.cpp index df8f8f411..a89d163d6 100644 --- a/src/common/backend/utils/cache/knl_globalsystupcache.cpp +++ b/src/common/backend/utils/cache/knl_globalsystupcache.cpp @@ -794,7 +794,7 @@ GlobalCatCTup *GlobalSysTupCache::SearchTupleMiss(InsertCatTupInfo *tup_info) if (!tup_info->has_concurrent_lock) { tup_info->canInsertGSC = false; } else { - tup_info->canInsertGSC = CanTupleInertGSC(ntp); + tup_info->canInsertGSC = CanTupleInsertGSC(ntp); if (!tup_info->canInsertGSC) { /* unlock concurrent immediately, any one can invalid cache now */ ReleaseGSCTableReadLock(&tup_info->has_concurrent_lock, m_concurrent_lock); @@ -811,7 +811,7 @@ GlobalCatCTup *GlobalSysTupCache::SearchTupleMiss(InsertCatTupInfo *tup_info) if (!tup_info->has_concurrent_lock) { tup_info->canInsertGSC = false; } else { - tup_info->canInsertGSC = CanTupleInertGSC(ntp); + tup_info->canInsertGSC = CanTupleInsertGSC(ntp); if (!tup_info->canInsertGSC) { /* unlock concurrent immediately, any one can invalid cache now */ ReleaseGSCTableReadLock(&tup_info->has_concurrent_lock, m_concurrent_lock); @@ -1080,7 +1080,7 @@ GlobalCatCList *GlobalSysTupCache::SearchListMiss(InsertCatListInfo *list_info) * See if there's an entry for this tuple already. */ InitInsertCatTupInfo(&tup_info, ntp, list_info->arguments); - tup_info.canInsertGSC = list_info->has_concurrent_lock && CanTupleInertGSC(ntp); + tup_info.canInsertGSC = list_info->has_concurrent_lock && CanTupleInsertGSC(ntp); GlobalCatCTup *ct = InsertHeapTupleIntoCatCacheInList(&tup_info); list_info->canInsertGSC = list_info->canInsertGSC && ct->canInsertGSC; /* @@ -1849,7 +1849,7 @@ GlobalCatCTup *GlobalSysTupCache::SearchTupleMissWithArgModes(InsertCatTupInfo * if (!tup_info->has_concurrent_lock) { tup_info->canInsertGSC = false; } else { - tup_info->canInsertGSC = CanTupleInertGSC(ntp); + tup_info->canInsertGSC = CanTupleInsertGSC(ntp); if (!tup_info->canInsertGSC) { /* unlock concurrent immediately, any one can invalid cache now */ ReleaseGSCTableReadLock(&tup_info->has_concurrent_lock, m_concurrent_lock); diff --git a/src/include/utils/knl_globalsyscache_common.h b/src/include/utils/knl_globalsyscache_common.h index b4c6f2345..79901fe1c 100644 --- a/src/include/utils/knl_globalsyscache_common.h +++ b/src/include/utils/knl_globalsyscache_common.h @@ -42,7 +42,7 @@ * for tuple uncommitted or being deleted, we dont sure it whether it will be committed or aborted, * so just store them into localcatcache, and invalid them by si msg */ -inline bool CanTupleInertGSC(HeapTuple tuple) +inline bool CanTupleInsertGSC(HeapTuple tuple) { if (tuple->t_tableOid == InvalidOid) { // this is a heapformtuple From 2cd06dc5757fbb06a82c2e13fc4c09c406b2dd8f Mon Sep 17 00:00:00 2001 From: yaojun Date: Fri, 11 Aug 2023 10:35:00 +0800 Subject: [PATCH 122/304] fix comment for CanTupleInertGSC --- src/include/utils/knl_globalsyscache_common.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/include/utils/knl_globalsyscache_common.h b/src/include/utils/knl_globalsyscache_common.h index b4c6f2345..5d357c731 100644 --- a/src/include/utils/knl_globalsyscache_common.h +++ b/src/include/utils/knl_globalsyscache_common.h @@ -39,8 +39,8 @@ /* - * for tuple uncommitted or being deleted, we dont sure it whether it will be committed or aborted, - * so just store them into localcatcache, and invalid them by si msg + * for tuple uncommitted or being deleted, we don't sure whether it will be committed or aborted, + * so just store them into local cache, and invalid them by send message */ inline bool CanTupleInertGSC(HeapTuple tuple) { From 8786eb407697f780c51078c26eaf81cafd4216c9 Mon Sep 17 00:00:00 2001 From: "changying.yue" Date: Fri, 11 Aug 2023 06:49:32 +0000 Subject: [PATCH 123/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=20=E5=BD=93=E6=B8=B8?= =?UTF-8?q?=E6=A0=87=E4=BD=BF=E7=94=A8=E5=A4=8D=E5=90=88=E7=B1=BB=E5=9E=8B?= =?UTF-8?q?=E5=9C=A8=E5=BC=82=E5=B8=B8=E5=9C=BA=E6=99=AF=E4=B8=8B=E5=AF=BC?= =?UTF-8?q?=E8=87=B4OG=20core=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../storage/access/transam/xact.cpp | 2 + .../expected/plpgsql_cursor_rowtype.out | 59 +++++++++++++++++++ .../regress/sql/plpgsql_cursor_rowtype.sql | 36 +++++++++++ 3 files changed, 97 insertions(+) diff --git a/src/gausskernel/storage/access/transam/xact.cpp b/src/gausskernel/storage/access/transam/xact.cpp index 8af3cbff8..aae577d6e 100755 --- a/src/gausskernel/storage/access/transam/xact.cpp +++ b/src/gausskernel/storage/access/transam/xact.cpp @@ -3900,6 +3900,8 @@ static void AbortTransaction(bool PerfectRollback, bool STP_rollback) /* reset flag is_delete_function */ u_sess->plsql_cxt.is_delete_function = false; + list_free_ext(u_sess->plsql_cxt.CursorRecordTypeList); + /* * do abort processing */ diff --git a/src/test/regress/expected/plpgsql_cursor_rowtype.out b/src/test/regress/expected/plpgsql_cursor_rowtype.out index 8b7665a1f..942d5e195 100644 --- a/src/test/regress/expected/plpgsql_cursor_rowtype.out +++ b/src/test/regress/expected/plpgsql_cursor_rowtype.out @@ -959,6 +959,65 @@ fetch c3; (1 row) close c3; +---- 不在 TRANSACTION Block里的游标声明导致 core的问题 +--游标依赖row type，后续alter type +drop type if exists type_cursor_bugfix_0001; +NOTICE: type "type_cursor_bugfix_0001" does not exist, skipping +create type type_cursor_bugfix_0001 as (a int, b int); +--游标依赖type，alter type报错 +begin; +declare c5 cursor for select (i,2^30)::type_cursor_bugfix_0001 from generate_series(1,10) i; +fetch c5; + row +---------------- + (1,1073741824) +(1 row) + +fetch c5; + row +---------------- + (2,1073741824) +(1 row) + +alter type type_cursor_bugfix_0001 alter attribute b type text;--error +ERROR: cannot ALTER TABLE "type_cursor_bugfix_0001" because it is being used by active queries in this session +end; +/ +--close后，可以成功alter +begin; +ERROR: syntax error at or near "/" +LINE 1: / + ^ +declare c7 cursor for select (i,2^30)::type_cursor_bugfix_0001 from generate_series(1,10) i; +ERROR: DECLARE CURSOR can only be used in transaction blocks +fetch c7; +ERROR: cursor "c7" does not exist +fetch c7; +ERROR: cursor "c7" does not exist +close c7; +ERROR: cursor "c7" does not exist +alter type type_cursor_bugfix_0001 alter attribute b type text;--success +declare c8 cursor for select (i,2^30)::type_cursor_bugfix_0001 from generate_series(1,10) i; +ERROR: DECLARE CURSOR can only be used in transaction blocks +fetch c8; +ERROR: cursor "c8" does not exist +fetch c8; +ERROR: cursor "c8" does not exist +rollback; +NOTICE: there is no transaction in progress +/ +begin; +ERROR: syntax error at or near "/" +LINE 1: / + ^ +cursor c9 for select (i,2^30)::type_cursor_bugfix_0001 from generate_series(1,10) i; +ERROR: DECLARE CURSOR can only be used in transaction blocks +close c9; +ERROR: cursor "c9" does not exist +alter type type_cursor_bugfix_0001 alter attribute b type text;--success +end; +WARNING: there is no transaction in progress +drop type if exists type_cursor_bugfix_0001; ---- clean ---- drop package pck1; NOTICE: drop cascades to function plpgsql_cursor_rowtype.p1() diff --git a/src/test/regress/sql/plpgsql_cursor_rowtype.sql b/src/test/regress/sql/plpgsql_cursor_rowtype.sql index 7698c70e6..21231b6ef 100644 --- a/src/test/regress/sql/plpgsql_cursor_rowtype.sql +++ b/src/test/regress/sql/plpgsql_cursor_rowtype.sql @@ -758,6 +758,42 @@ alter type foo alter attribute b type text;--success fetch c3; close c3; +---- 不在 TRANSACTION Block里的游标声明导致 core的问题 +--游标依赖row type，后续alter type +drop type if exists type_cursor_bugfix_0001; +create type type_cursor_bugfix_0001 as (a int, b int); + +--游标依赖type，alter type报错 +begin; +declare c5 cursor for select (i,2^30)::type_cursor_bugfix_0001 from generate_series(1,10) i; +fetch c5; +fetch c5; +alter type type_cursor_bugfix_0001 alter attribute b type text;--error +end; +/ + +--close后，可以成功alter +begin; +declare c7 cursor for select (i,2^30)::type_cursor_bugfix_0001 from generate_series(1,10) i; +fetch c7; +fetch c7; +close c7; +alter type type_cursor_bugfix_0001 alter attribute b type text;--success +declare c8 cursor for select (i,2^30)::type_cursor_bugfix_0001 from generate_series(1,10) i; +fetch c8; +fetch c8; +rollback; +/ + +begin; +cursor c9 for select (i,2^30)::type_cursor_bugfix_0001 from generate_series(1,10) i; +close c9; +alter type type_cursor_bugfix_0001 alter attribute b type text;--success +end; + +drop type if exists type_cursor_bugfix_0001; + + ---- clean ---- drop package pck1; drop package pck2; From e1b92cf6d72d84b3d5f4e3124b42f137fb09db91 Mon Sep 17 00:00:00 2001 From: zhangchao Date: Fri, 11 Aug 2023 17:17:22 +0800 Subject: [PATCH 124/304] =?UTF-8?q?=E6=94=AF=E6=8C=81ldap=E8=AE=A4?= =?UTF-8?q?=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/interfaces/libpq/fe-auth.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/common/interfaces/libpq/fe-auth.cpp b/src/common/interfaces/libpq/fe-auth.cpp index fc56b5099..1e7773e2d 100644 --- a/src/common/interfaces/libpq/fe-auth.cpp +++ b/src/common/interfaces/libpq/fe-auth.cpp @@ -1040,9 +1040,11 @@ static int pg_password_sendauth(PGconn* conn, const char* password, AuthRequest break; } +#ifdef USE_LDAP case AUTH_REQ_PASSWORD: pwd_to_send = password; break; +#endif /* * Notice: Authentication of send password directly are not currently supported. * need to: We remove the branch of AUTH_REQ_PASSWORD here for both implication and @@ -1211,7 +1213,9 @@ int pg_fe_sendauth(AuthRequest areq, PGconn* conn) case AUTH_REQ_MD5: case AUTH_REQ_MD5_SHA256: +#ifdef USE_LDAP case AUTH_REQ_PASSWORD: +#endif case AUTH_REQ_SHA256: #ifdef ENABLE_LITE_MODE case AUTH_REQ_SHA256_RFC: From 10afc18f1f3ce31624d23a95f17b96aa9547b0b0 Mon Sep 17 00:00:00 2001 From: lukeman Date: Sat, 12 Aug 2023 15:33:41 +0800 Subject: [PATCH 125/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E9=97=AE=E9=A2=98?= =?UTF-8?q?=EF=BC=9Areplace(string,substring)=E5=87=BD=E6=95=B02=E4=B8=AA?= =?UTF-8?q?=E5=8F=82=E6=95=B0=E4=B8=80=E6=A0=B7=E6=97=B6=E6=8A=A5=E9=94=99?= =?UTF-8?q?=EF=BC=8C=E9=A2=84=E6=9C=9F=E8=BF=94=E5=9B=9Enull?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/utils/adt/varlena.cpp | 18 ++++++++++++------ .../regress/expected/not_accept_empty_str.out | 17 +++++++++++++++-- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/src/common/backend/utils/adt/varlena.cpp b/src/common/backend/utils/adt/varlena.cpp index 40dcb0f66..129277cce 100644 --- a/src/common/backend/utils/adt/varlena.cpp +++ b/src/common/backend/utils/adt/varlena.cpp @@ -4323,14 +4323,20 @@ Datum replace_text_with_two_args(PG_FUNCTION_ARGS) { if (PG_ARGISNULL(0)) PG_RETURN_NULL(); - if (PG_ARGISNULL(1)) PG_RETURN_TEXT_P(PG_GETARG_TEXT_PP(0)); - - return DirectFunctionCall3(replace_text, - PG_GETARG_DATUM(0), - PG_GETARG_DATUM(1), - CStringGetTextDatum("\0")); + FunctionCallInfoData locfcinfo; + Datum result; + InitFunctionCallInfoData(locfcinfo, NULL, 3, InvalidOid, NULL, NULL); + locfcinfo.arg[0] = PG_GETARG_DATUM(0); + locfcinfo.arg[1] = PG_GETARG_DATUM(1); + locfcinfo.arg[2] = CStringGetTextDatum("\0"); + locfcinfo.argnull[0] = false; + locfcinfo.argnull[1] = false; + locfcinfo.argnull[2] = false; + result = (*replace_text)(&locfcinfo); + fcinfo->isnull = locfcinfo.isnull; + return result; } /* diff --git a/src/test/regress/expected/not_accept_empty_str.out b/src/test/regress/expected/not_accept_empty_str.out index a2149a902..6cf91f8d4 100644 --- a/src/test/regress/expected/not_accept_empty_str.out +++ b/src/test/regress/expected/not_accept_empty_str.out @@ -885,7 +885,11 @@ SELECT replace('abc', 'ab') is null; (1 row) SELECT replace('abc', 'abc') is null; -ERROR: function returned NULL + ?column? +---------- + t +(1 row) + SELECT replace('abc', '') is null; ?column? ---------- @@ -1781,7 +1785,16 @@ SELECT a as p1, b as p2, c as p3, replace(a, b, c) is null from replace3; create table replace2 (a text, b text); insert into replace2 values('abc', 'ab'), ('abc', 'abc'), ('abc', ''), ('abc', null), ('', 'ab'), (null, 'ab'); SELECT a as p1, b as p2, replace(a, b) is null from replace2; -ERROR: function returned NULL + p1 | p2 | ?column? +-----+-----+---------- + abc | ab | f + abc | abc | t + abc | | f + abc | | f + | ab | t + | ab | t +(6 rows) + create table split_part3 (a text, b text, c int); insert into split_part3 values('1~2~3', '~', 3), ('1~2~3~', '~', 4), ('1~2~3', '', 1), ('1~2~3', null, 1), ('', '~', 1), (null, '~', 1); SELECT a as p1, b as p2, c as p3, split_part(a, b, c) is null from split_part3; From b2b04c1764c3e4f30ab4374e26074ba55fbe53dd Mon Sep 17 00:00:00 2001 From: chendong76 <1209756284@qq.com> Date: Wed, 2 Aug 2023 16:18:53 +0800 Subject: [PATCH 126/304] =?UTF-8?q?[bugfix]=E8=A7=A3=E5=86=B3=E6=8C=89?= =?UTF-8?q?=E9=9C=80=E5=9B=9E=E6=94=BE=E9=83=A8=E5=88=86=E9=97=AE=E9=A2=98?= =?UTF-8?q?=EF=BC=9A1.=E9=83=A8=E5=88=86=E6=97=A5=E5=BF=97=E5=9B=9E?= =?UTF-8?q?=E6=94=BE=E9=80=89=E6=8B=A9=E9=94=99=E8=AF=AF=E9=97=AE=E9=A2=98?= =?UTF-8?q?=EF=BC=9B2.=E6=8C=89=E9=9C=80=E5=9B=9E=E6=94=BE=E6=9C=AA?= =?UTF-8?q?=E6=AD=A3=E5=B8=B8=E7=BB=93=E6=9D=9F=E5=90=8E=E5=A4=9A=E6=AC=A1?= =?UTF-8?q?=E9=87=8D=E5=90=AF=EF=BC=8C=E5=AF=BC=E8=87=B4=E6=89=BE=E4=B8=8D?= =?UTF-8?q?=E5=88=B0redo=E7=82=B9=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../transam/ondemand_extreme_rto/page_redo.cpp | 13 +++++++------ src/gausskernel/storage/access/transam/xlog.cpp | 8 ++++---- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/gausskernel/storage/access/transam/ondemand_extreme_rto/page_redo.cpp b/src/gausskernel/storage/access/transam/ondemand_extreme_rto/page_redo.cpp index 51e880f1f..4456afbd5 100644 --- a/src/gausskernel/storage/access/transam/ondemand_extreme_rto/page_redo.cpp +++ b/src/gausskernel/storage/access/transam/ondemand_extreme_rto/page_redo.cpp @@ -1074,14 +1074,10 @@ static void OnDemandPageManagerRedoSegParseState(XLogRecParseState *preState) Assert(g_redoWorker->slotId == 0); switch (preState->blockparse.blockhead.block_valid) { case BLOCK_DATA_SEG_EXTEND: - GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_4]); OnDemandPageManagerProcSegPipeLineSyncState(preState); - CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_4]); break; case BLOCK_DATA_SEG_FULL_SYNC_TYPE: - GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_8]); OnDemandPageManagerProcSegFullSyncState(preState); - CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_8]); break; case BLOCK_DATA_SEG_FILE_EXTEND_TYPE: default: @@ -1133,14 +1129,19 @@ void PageManagerRedoParseState(XLogRecParseState *preState) XLogBlockParseStateRelease(preState); break; case BLOCK_DATA_CREATE_DATABASE_TYPE: - case BLOCK_DATA_SEG_FILE_EXTEND_TYPE: GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); - OnDemandPageManagerRedoSegParseState(preState); + RedoPageManagerDistributeBlockRecord(hashMap, NULL); + /* wait until queue empty */ + WaitCurrentPipeLineRedoWorkersQueueEmpty(); + /* do atcual action */ + RedoPageManagerSyncDdlAction(preState); CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); break; + case BLOCK_DATA_SEG_FILE_EXTEND_TYPE: case BLOCK_DATA_SEG_FULL_SYNC_TYPE: GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_8]); OnDemandPageManagerRedoSegParseState(preState); + CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_8]); break; case BLOCK_DATA_CREATE_TBLSPC_TYPE: GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_7]); diff --git a/src/gausskernel/storage/access/transam/xlog.cpp b/src/gausskernel/storage/access/transam/xlog.cpp index 6a01a780b..c49404daf 100755 --- a/src/gausskernel/storage/access/transam/xlog.cpp +++ b/src/gausskernel/storage/access/transam/xlog.cpp @@ -10336,7 +10336,7 @@ void StartupXLOG(void) t_thrd.xlog_cxt.InRecovery = false; g_instance.roach_cxt.isRoachRestore = false; - if (!SS_STANDBY_FAILOVER && !SS_STANDBY_PROMOTING && !SS_IN_ONDEMAND_RECOVERY) { + if (!SS_STANDBY_FAILOVER && !SS_STANDBY_PROMOTING && !SS_IN_ONDEMAND_RECOVERY && SSOndemandRecoveryExitNormal) { LWLockAcquire(ControlFileLock, LW_EXCLUSIVE); t_thrd.shemem_ptr_cxt.ControlFile->state = DB_IN_PRODUCTION; t_thrd.shemem_ptr_cxt.ControlFile->time = (pg_time_t)time(NULL); @@ -10464,14 +10464,14 @@ void StartupXLOG(void) } } - if (SS_STANDBY_FAILOVER || SS_STANDBY_PROMOTING) { + if (SS_STANDBY_FAILOVER || SS_STANDBY_PROMOTING || !SSOndemandRecoveryExitNormal) { if (SS_STANDBY_FAILOVER) { g_instance.dms_cxt.SSRecoveryInfo.failover_ckpt_status = ALLOW_CKPT; pg_memory_barrier(); } if (!SS_IN_ONDEMAND_RECOVERY) { ereport(LOG, (errmodule(MOD_DMS), - errmsg("[SS switchover/SS failover] standby promoting: start full checkpoint."))); + errmsg("[SS switchover/SS failover/SS normal reform] start full checkpoint."))); RequestCheckpoint(CHECKPOINT_FORCE | CHECKPOINT_IMMEDIATE | CHECKPOINT_WAIT); LWLockAcquire(ControlFileLock, LW_EXCLUSIVE); t_thrd.shemem_ptr_cxt.ControlFile->state = DB_IN_PRODUCTION; @@ -10480,7 +10480,7 @@ void StartupXLOG(void) LWLockRelease(ControlFileLock); SSRecheckBufferPool(); ereport(LOG, (errmodule(MOD_DMS), - errmsg("[SS switchover/SS failover] standby promoting: finished full checkpoint" + errmsg("[SS switchover/SS failover/SS normal reform] finished full checkpoint" "and update control file"))); } } From 956b3c7bbfc4e2f6da40b95462c86779c4110016 Mon Sep 17 00:00:00 2001 From: liuzhanfeng2 Date: Sun, 13 Aug 2023 14:23:41 +0800 Subject: [PATCH 127/304] =?UTF-8?q?=E9=80=82=E9=85=8DDSS=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E4=B8=8E=E9=94=99=E8=AF=AF=E7=A0=81=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/pg_probackup/pg_probackup.cpp | 5 ++- src/bin/pg_resetxlog/pg_resetxlog.cpp | 3 +- src/common/backend/utils/cache/relmapper.cpp | 3 +- src/common/backend/utils/init/miscinit.cpp | 3 +- .../ddes/adapter/ss_dms_recovery.cpp | 5 ++- src/gausskernel/storage/dss/dss_adaptor.cpp | 3 -- src/gausskernel/storage/dss/fio_dss.cpp | 40 ++++--------------- src/gausskernel/storage/replication/slot.cpp | 3 +- .../replication/ss_cluster_replication.cpp | 11 +++-- src/include/storage/dss/dss_adaptor.h | 6 --- src/include/storage/dss/fio_dss.h | 3 -- src/include/storage/file/fio_device.h | 23 ++++++----- 12 files changed, 42 insertions(+), 66 deletions(-) diff --git a/src/bin/pg_probackup/pg_probackup.cpp b/src/bin/pg_probackup/pg_probackup.cpp index 64da31943..2d53f5640 100644 --- a/src/bin/pg_probackup/pg_probackup.cpp +++ b/src/bin/pg_probackup/pg_probackup.cpp @@ -953,12 +953,13 @@ static void dss_init(void) parse_vgname_args(instance_config.dss.vgname); /* Check dss connect */ - if (!dss_exist_dir(instance_config.dss.vgdata)) { + struct stat st; + if (stat(instance_config.dss.vgdata, &st) != 0 || !S_ISDIR(st.st_mode)) { elog(ERROR, "Could not connect dssserver, vgdata: \"%s\", socketpath: \"%s\", check and retry later.", instance_config.dss.vgdata, instance_config.dss.socketpath); } - if (strlen(instance_config.dss.vglog) && !dss_exist_dir(instance_config.dss.vglog)) { + if (strlen(instance_config.dss.vglog) && (stat(instance_config.dss.vglog, &st) != 0 || !S_ISDIR(st.st_mode))) { elog(ERROR, "Could not connect dssserver, vglog: \"%s\", socketpath: \"%s\", check and retry later.", instance_config.dss.vglog, instance_config.dss.socketpath); } diff --git a/src/bin/pg_resetxlog/pg_resetxlog.cpp b/src/bin/pg_resetxlog/pg_resetxlog.cpp index 9b7d27fa2..41ff527c4 100644 --- a/src/bin/pg_resetxlog/pg_resetxlog.cpp +++ b/src/bin/pg_resetxlog/pg_resetxlog.cpp @@ -490,7 +490,8 @@ static void SetGlobalDssParam(void) securec_check_c(rc, "\0", "\0"); XLogSegmentSize = DSS_XLOG_SEG_SIZE; /* Check dss connect */ - if (!dss_exist_dir(g_datadir.dss_data)) { + struct stat st; + if (stat(g_datadir.dss_data, &st) != 0 || !S_ISDIR(st.st_mode)) { fprintf(stderr, _("Could not connect dssserver, vgname: \"%s\", socketpath: \"%s\", \n" "please check that whether the dssserver is manually started and retry later.\n"), dss.vgname, dss.socketpath); diff --git a/src/common/backend/utils/cache/relmapper.cpp b/src/common/backend/utils/cache/relmapper.cpp index 4fe12d60a..594ca8801 100644 --- a/src/common/backend/utils/cache/relmapper.cpp +++ b/src/common/backend/utils/cache/relmapper.cpp @@ -493,7 +493,8 @@ void RelationMapFinishBootstrap(void) g_instance.attr.attr_storage.dss_attr.ss_dss_vg_name, RELMAPPER_FILENAME); securec_check_ss_c(rc, "\0", "\0"); - if (dss_exist_file(map_file_name)) { + struct stat st; + if (stat(map_file_name, &st) == 0 && S_ISREG(st.st_mode)) { return; } } diff --git a/src/common/backend/utils/init/miscinit.cpp b/src/common/backend/utils/init/miscinit.cpp index b8e974640..329706f92 100644 --- a/src/common/backend/utils/init/miscinit.cpp +++ b/src/common/backend/utils/init/miscinit.cpp @@ -2160,7 +2160,8 @@ void initDSSConf(void) } // check whether dss connect is successful. - if (!dss_exist_dir(g_instance.attr.attr_storage.dss_attr.ss_dss_vg_name)) { + struct stat st; + if (stat(g_instance.attr.attr_storage.dss_attr.ss_dss_vg_name, &st) != 0 || !S_ISDIR(st.st_mode)) { ereport(FATAL, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("Could not connect dssserver, vgname: \"%s\", socketpath: \"%s\"", g_instance.attr.attr_storage.dss_attr.ss_dss_vg_name, diff --git a/src/gausskernel/ddes/adapter/ss_dms_recovery.cpp b/src/gausskernel/ddes/adapter/ss_dms_recovery.cpp index 48dd3646b..b8a5a2c62 100644 --- a/src/gausskernel/ddes/adapter/ss_dms_recovery.cpp +++ b/src/gausskernel/ddes/adapter/ss_dms_recovery.cpp @@ -167,7 +167,8 @@ void SSInitReformerControlPages(void) /* * If already exists control file, reformer page must have been initialized */ - if (dss_exist_file(XLOG_CONTROL_FILE)) { + struct stat st; + if (stat(XLOG_CONTROL_FILE, &st) == 0 && S_ISREG(st.st_mode)) { SSReadControlFile(REFORM_CTRL_PAGE); if (g_instance.dms_cxt.SSReformerControl.list_stable != 0 || g_instance.dms_cxt.SSReformerControl.primaryInstId == SS_MY_INST_ID) { @@ -187,7 +188,7 @@ void SSInitReformerControlPages(void) * Initialize list_stable and primaryInstId * First node to initdb is chosen as primary for now, and for first-time cluster startup. */ - Assert(!dss_exist_file(XLOG_CONTROL_FILE)); + Assert(stat(XLOG_CONTROL_FILE, &st) != 0 || !S_ISREG(st.st_mode)); g_instance.dms_cxt.SSReformerControl.list_stable = 0; g_instance.dms_cxt.SSReformerControl.primaryInstId = SS_MY_INST_ID; g_instance.dms_cxt.SSReformerControl.recoveryInstId = INVALID_INSTANCEID; diff --git a/src/gausskernel/storage/dss/dss_adaptor.cpp b/src/gausskernel/storage/dss/dss_adaptor.cpp index 6ca556a17..0b1ce5c67 100644 --- a/src/gausskernel/storage/dss/dss_adaptor.cpp +++ b/src/gausskernel/storage/dss/dss_adaptor.cpp @@ -104,9 +104,7 @@ int dss_device_init(const char *conn_path, bool enable_dss) SS_RETURN_IFERR(dss_load_symbol(device_op.handle, "dss_ftruncate", (void **)&device_op.dss_truncate)); SS_RETURN_IFERR(dss_load_symbol(device_op.handle, "dss_fwrite", (void **)&device_op.dss_write)); SS_RETURN_IFERR(dss_load_symbol(device_op.handle, "dss_pwrite", (void **)&device_op.dss_pwrite)); - SS_RETURN_IFERR(dss_load_symbol(device_op.handle, "dss_fexist", (void **)&device_op.dss_exist)); SS_RETURN_IFERR(dss_load_symbol(device_op.handle, "dss_dmake", (void **)&device_op.dss_create_dir)); - SS_RETURN_IFERR(dss_load_symbol(device_op.handle, "dss_dexist", (void **)&device_op.dss_exist_dir)); SS_RETURN_IFERR(dss_load_symbol(device_op.handle, "dss_dopen", (void **)&device_op.dss_open_dir)); SS_RETURN_IFERR(dss_load_symbol(device_op.handle, "dss_dread", (void **)&device_op.dss_read_dir)); SS_RETURN_IFERR(dss_load_symbol(device_op.handle, "dss_dclose", (void **)&device_op.dss_close_dir)); @@ -119,7 +117,6 @@ int dss_device_init(const char *conn_path, bool enable_dss) SS_RETURN_IFERR(dss_load_symbol(device_op.handle, "dss_get_error", (void **)&device_op.dss_get_error)); SS_RETURN_IFERR(dss_load_symbol(device_op.handle, "dss_symlink", (void **)&device_op.dss_link)); SS_RETURN_IFERR(dss_load_symbol(device_op.handle, "dss_unlink", (void **)&device_op.dss_unlink)); - SS_RETURN_IFERR(dss_load_symbol(device_op.handle, "dss_islink", (void **)&device_op.dss_exist_link)); SS_RETURN_IFERR(dss_load_symbol(device_op.handle, "dss_readlink", (void **)&device_op.dss_read_link)); SS_RETURN_IFERR(dss_load_symbol(device_op.handle, "dss_stat", (void **)&device_op.dss_stat)); SS_RETURN_IFERR(dss_load_symbol(device_op.handle, "dss_lstat", (void **)&device_op.dss_lstat)); diff --git a/src/gausskernel/storage/dss/fio_dss.cpp b/src/gausskernel/storage/dss/fio_dss.cpp index ae7673ccd..3feac456c 100644 --- a/src/gausskernel/storage/dss/fio_dss.cpp +++ b/src/gausskernel/storage/dss/fio_dss.cpp @@ -93,15 +93,6 @@ void dss_set_errno(int *errcode) } } -bool dss_exist_file(const char *file_name) -{ - bool result = false; - if (g_dss_device_op.dss_exist(file_name, &result) != DSS_SUCCESS) { - dss_set_errno(NULL); - return false; - } - return result; -} int dss_access_file(const char *file_name, int mode) { @@ -118,16 +109,6 @@ int dss_create_dir(const char *name, mode_t mode) return GS_SUCCESS; } -bool dss_exist_dir(const char *name) -{ - bool result = false; - if (g_dss_device_op.dss_exist_dir(name, &result) != DSS_SUCCESS) { - dss_set_errno(NULL); - return false; - } - return result; -} - int dss_open_dir(const char *name, DIR **dir_handle) { DSS_DIR *dss_dir = NULL; @@ -216,9 +197,10 @@ int dss_remove_file(const char *name) int dss_open_file(const char *name, int flags, mode_t mode, int *handle) { - if ((flags & O_CREAT) != 0 && !dss_exist_file(name)) { + struct stat st; + if ((flags & O_CREAT) != 0 && dss_stat_file(name, &st) != GS_SUCCESS) { // file not exists, create it first. - if (g_dss_device_op.dss_create(name, flags) != DSS_SUCCESS) { + if (errno == ERR_DSS_FILE_NOT_EXIST && g_dss_device_op.dss_create(name, flags) != DSS_SUCCESS) { dss_set_errno(NULL); return GS_ERROR; } @@ -508,16 +490,6 @@ int dss_unlink_target(const char *name) return GS_SUCCESS; } -bool dss_exist_link(const char *name) -{ - bool result = false; - if (g_dss_device_op.dss_exist_link(name, &result) != DSS_SUCCESS) { - dss_set_errno(NULL); - return false; - } - return result; -} - ssize_t dss_read_link(const char *path, char *buf, size_t buf_size) { ssize_t result = (ssize_t)g_dss_device_op.dss_read_link(path, buf, buf_size); @@ -696,9 +668,11 @@ int dss_set_server_status_wrapper() int dss_remove_dev(const char *name) { - if (dss_exist_file(name)) { + struct stat st; + int ret = lstat(name, &st); + if (ret == 0 && S_ISREG(st.st_mode)) { return dss_remove_file(name); - } else if (dss_exist_link(name)) { + } else if (ret == 0 && S_ISLNK(st.st_mode)) { return dss_unlink_target(name); } else { return GS_SUCCESS; diff --git a/src/gausskernel/storage/replication/slot.cpp b/src/gausskernel/storage/replication/slot.cpp index d3d31edcd..c26f39ceb 100644 --- a/src/gausskernel/storage/replication/slot.cpp +++ b/src/gausskernel/storage/replication/slot.cpp @@ -2635,7 +2635,8 @@ static bool CheckExistReplslotPath(char *path) errno_t rc = strcat_s(path_for_check, MAXPGPATH, suffix); securec_check_ss(rc, "\0", "\0"); } - if (dss_exist_dir(path_for_check)) { + struct stat st; + if (stat(path_for_check, &st) == 0 && S_ISDIR(st.st_mode)) { return true; } diff --git a/src/gausskernel/storage/replication/ss_cluster_replication.cpp b/src/gausskernel/storage/replication/ss_cluster_replication.cpp index 3283018b9..ca1f33cef 100644 --- a/src/gausskernel/storage/replication/ss_cluster_replication.cpp +++ b/src/gausskernel/storage/replication/ss_cluster_replication.cpp @@ -30,7 +30,8 @@ void WriteSSDoradoCtlInfoFile() { - Assert(dss_exist_file(SS_DORADO_CTRL_FILE)); + struct stat st; + Assert(stat(SS_DORADO_CTRL_FILE, &st) == 0 && S_ISREG(st.st_mode)); Assert(g_instance.xlog_cxt.ssReplicationXLogCtl != NULL); ShareStorageXLogCtl *ctlInfo = g_instance.xlog_cxt.ssReplicationXLogCtl; errno_t errorno = EOK; @@ -65,7 +66,8 @@ void WriteSSDoradoCtlInfoFile() void ReadSSDoradoCtlInfoFile() { - Assert(dss_exist_file(SS_DORADO_CTRL_FILE)); + struct stat st; + Assert(stat(SS_DORADO_CTRL_FILE, &st) == 0 && S_ISREG(st.st_mode)); Assert(g_instance.xlog_cxt.ssReplicationXLogCtl != NULL); ShareStorageXLogCtl *ctlInfo = g_instance.xlog_cxt.ssReplicationXLogCtl; errno_t errorno = EOK; @@ -105,7 +107,8 @@ void ReadSSDoradoCtlInfoFile() void InitSSDoradoCtlInfoFile() { - if (dss_exist_file(SS_DORADO_CTRL_FILE)) { + struct stat st; + if (stat(SS_DORADO_CTRL_FILE, &st) == 0 && S_ISREG(st.st_mode)) { ReadSSDoradoCtlInfoFile(); ereport(LOG, (errcode_for_file_access(), errmsg("[InitSSDoradoCtlInfoFile] Dorado ctl info file already exists."))); return; @@ -117,7 +120,7 @@ void InitSSDoradoCtlInfoFile() int fd = -1; char buffer[SS_DORADO_CTL_INFO_SIZE] __attribute__((__aligned__(ALIGNOF_BUFFER))); /* need to be aligned */ errno_t errorno = EOK; - Assert(!dss_exist_file(SS_DORADO_CTRL_FILE)); + Assert(stat(SS_DORADO_CTRL_FILE, &st) !=0 || !S_ISREG(st.st_mode)); /* create SS_DORADO_CTRL_FILE first time */ fd = BasicOpenFile(SS_DORADO_CTRL_FILE, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, S_IRUSR | S_IWUSR); diff --git a/src/include/storage/dss/dss_adaptor.h b/src/include/storage/dss/dss_adaptor.h index d8bac4902..c46b3dbe6 100644 --- a/src/include/storage/dss/dss_adaptor.h +++ b/src/include/storage/dss/dss_adaptor.h @@ -44,9 +44,7 @@ typedef int (*dss_trucate_device)(int handle, long keep_size); typedef int (*dss_create_device)(const char *name, int flags); typedef int (*dss_remove_device)(const char *name); typedef int (*dss_close_device)(int handle); -typedef int (*dss_exist_device)(const char *name, bool *result); typedef int (*dss_create_device_dir)(const char *name); -typedef int (*dss_exist_device_dir)(const char *name, bool *result); typedef dss_dir_handle (*dss_open_device_dir)(const char *name); typedef int (*dss_read_device_dir)(dss_dir_handle dir, dss_dirent_t *item, dss_dir_item_t *result); typedef int (*dss_close_device_dir)(dss_dir_handle dir); @@ -56,7 +54,6 @@ typedef int (*dss_check_device_size)(int size); typedef int (*dss_align_device_size)(int size); typedef int (*dss_link_device)(const char *oldpath, const char *newpath); typedef int (*dss_unlink_device)(const char *path); -typedef int (*dss_exist_device_link)(const char *path, bool *result); typedef int (*dss_device_name)(int handle, char *fname, size_t fname_size); typedef int (*dss_read_device_link)(const char *path, char *buf, int bufsize); typedef int (*dss_stat_device)(const char *path, dss_stat_info_t item); @@ -88,9 +85,7 @@ typedef struct st_dss_device_op_t { dss_seek_device dss_seek; dss_trucate_device dss_truncate; dss_close_device dss_close; - dss_exist_device dss_exist; dss_create_device_dir dss_create_dir; - dss_exist_device_dir dss_exist_dir; dss_rename_device dss_rename; dss_check_device_size dss_check_size; dss_align_device_size dss_align_size; @@ -103,7 +98,6 @@ typedef struct st_dss_device_op_t { dss_remove_device_dir dss_remove_dir; dss_link_device dss_link; dss_unlink_device dss_unlink; - dss_exist_device_link dss_exist_link; dss_read_device_link dss_read_link; dss_stat_device dss_stat; dss_lstat_device dss_lstat; diff --git a/src/include/storage/dss/fio_dss.h b/src/include/storage/dss/fio_dss.h index 585047dfd..0dd0e4c63 100644 --- a/src/include/storage/dss/fio_dss.h +++ b/src/include/storage/dss/fio_dss.h @@ -34,10 +34,8 @@ void dss_device_register(dss_device_op_t *dss_device_op, bool enable_dss); void dss_set_errno(int *errcode); -bool dss_exist_file(const char *file_name); int dss_access_file(const char *file_name, int mode); int dss_create_dir(const char *name, mode_t mode); -bool dss_exist_dir(const char *name); int dss_open_dir(const char *name, DIR **dir_handle); int dss_read_dir(DIR *dir_handle, struct dirent **result); int dss_close_dir(DIR *dir_handle); @@ -66,7 +64,6 @@ off_t dss_get_file_size(const char *fname); int dss_fallocate_file(int handle, int mode, off_t offset, off_t len); int dss_link(const char *src, const char *dst); int dss_unlink_target(const char *name); -bool dss_exist_link(const char *name); ssize_t dss_read_link(const char *path, char *buf, size_t buf_size); int dss_setvbuf(FILE *stream, char *buf, int mode, size_t size); int dss_feof(FILE *stream); diff --git a/src/include/storage/file/fio_device.h b/src/include/storage/file/fio_device.h index 105fbfb5d..b19048803 100644 --- a/src/include/storage/file/fio_device.h +++ b/src/include/storage/file/fio_device.h @@ -166,10 +166,10 @@ static inline int fallocate_dev(int fd, int mode, off_t offset, off_t len) static inline int access_dev(const char *pathname, int mode) { if (is_dss_file(pathname)) { - if (!dss_exist_file(pathname) && !dss_exist_dir(pathname)) { + if (dss_access_file(pathname, mode) != GS_SUCCESS) { return -1; } - return dss_access_file(pathname, mode); + return 0; } else { return access(pathname, mode); } @@ -215,7 +215,8 @@ static inline int symlink_dev(const char *target, const char *linkpath) static inline ssize_t readlink_dev(const char *pathname, char *buf, size_t bufsiz) { if (is_dss_file(pathname)) { - if (!dss_exist_link(pathname)) { + struct stat st; + if (dss_lstat_file(pathname, &st) != GS_SUCCESS || !S_ISLNK(st.st_mode)) { return -1; } @@ -249,11 +250,13 @@ static inline int unlink_dev(const char *pathname) static inline int lstat_dev(const char * pathname, struct stat * statbuf) { if (is_dss_file(pathname)) { - if (!dss_exist_file(pathname) && !dss_exist_dir(pathname)) { - errno = ENOENT; + if (dss_lstat_file(pathname, statbuf) != GS_SUCCESS) { + if (errno == ERR_DSS_FILE_NOT_EXIST) { + errno = ENOENT; + } return -1; } - return dss_lstat_file(pathname, statbuf); + return GS_SUCCESS; } else { return lstat(pathname, statbuf); } @@ -262,11 +265,13 @@ static inline int lstat_dev(const char * pathname, struct stat * statbuf) static inline int stat_dev(const char *pathname, struct stat *statbuf) { if (is_dss_file(pathname)) { - if (!dss_exist_file(pathname) && !dss_exist_dir(pathname)) { - errno = ENOENT; + if (dss_stat_file(pathname, statbuf) != GS_SUCCESS) { + if (errno == ERR_DSS_FILE_NOT_EXIST) { + errno = ENOENT; + } return -1; } - return dss_stat_file(pathname, statbuf); + return GS_SUCCESS; } else { return stat(pathname, statbuf); } From efb218de7769069d9b3960c4ffa60d28a6e99ab0 Mon Sep 17 00:00:00 2001 From: chenbd Date: Mon, 7 Aug 2023 16:05:45 +0800 Subject: [PATCH 128/304] fix create as in rewritehandler This reverts commit 41b5cbf3cdd607313b1dc6b9dc6b78ab2f7b1a3b. --- .../optimizer/rewrite/rewriteHandler.cpp | 23 ++++++++++++++++--- src/test/regress/expected/create_table_3.out | 12 ++++++++++ src/test/regress/sql/create_table_3.sql | 7 ++++++ 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/src/gausskernel/optimizer/rewrite/rewriteHandler.cpp b/src/gausskernel/optimizer/rewrite/rewriteHandler.cpp index 60b345ab9..ece65fcf5 100644 --- a/src/gausskernel/optimizer/rewrite/rewriteHandler.cpp +++ b/src/gausskernel/optimizer/rewrite/rewriteHandler.cpp @@ -35,6 +35,7 @@ #include "parser/parsetree.h" #include "parser/parse_merge.h" #include "parser/parse_hint.h" +#include "parser/parse_type.h" #include "rewrite/rewriteDefine.h" #include "rewrite/rewriteHandler.h" #include "rewrite/rewriteManip.h" @@ -4507,6 +4508,21 @@ static bool findAttrByName(const char* attributeName, List* tableElts, int maxle } return false; } +/* + * for create table as in B foramt, add type's oid and typemod in tableElts + */ +static void addInitAttrType(List* tableElts) +{ + ListCell* lc = NULL; + + foreach (lc, tableElts) { + Node* node = (Node*)lfirst(lc); + if (IsA(node, ColumnDef)) { + ColumnDef* def = (ColumnDef*)node; + typenameTypeIdAndMod(NULL, def->typname, &def->typname->typeOid, &def->typname->typemod); + } + } +} char* GetCreateTableStmt(Query* parsetree, CreateTableAsStmt* stmt) { @@ -4517,8 +4533,10 @@ char* GetCreateTableStmt(Query* parsetree, CreateTableAsStmt* stmt) IntoClause* into = stmt->into; List* tableElts = NIL; - if (u_sess->attr.attr_sql.sql_compatibility == B_FORMAT) + if (u_sess->attr.attr_sql.sql_compatibility == B_FORMAT) { tableElts = stmt->into->tableElts; + addInitAttrType(tableElts); + } int initlen = list_length(tableElts); /* Obtain the target list of new table */ @@ -4674,8 +4692,7 @@ char* GetCreateTableStmt(Query* parsetree, CreateTableAsStmt* stmt) StringInfo cquery = makeStringInfo(); - if (u_sess->attr.attr_sql.sql_compatibility != B_FORMAT || initlen <= 0) - deparse_query(parsetree, cquery, NIL, false, false); + deparse_query(parsetree, cquery, NIL, false, false); return cquery->data; } diff --git a/src/test/regress/expected/create_table_3.out b/src/test/regress/expected/create_table_3.out index ffe064d88..4d28ae444 100644 --- a/src/test/regress/expected/create_table_3.out +++ b/src/test/regress/expected/create_table_3.out @@ -218,5 +218,17 @@ select * from tb_unique; 1 | 1 (3 rows) +--fixbug +drop table if exists t_base; +create table t_base(col1 int, col2 int, col3 int); +insert into t_base values(1,2,3),(11,22,33); +create table ttt3(col int) as select * from t_base; +select * from ttt3; + col | col1 | col2 | col3 +-----+------+------+------ + | 1 | 2 | 3 + | 11 | 22 | 33 +(2 rows) + \c postgres drop database b_createas; diff --git a/src/test/regress/sql/create_table_3.sql b/src/test/regress/sql/create_table_3.sql index 7aca17402..7301e9023 100644 --- a/src/test/regress/sql/create_table_3.sql +++ b/src/test/regress/sql/create_table_3.sql @@ -107,5 +107,12 @@ insert into tb_unique values(1,1) ON DUPLICATE KEY UPDATE a = 1, b = 1; select * from tb_primary; select * from tb_unique; +--fixbug +drop table if exists t_base; +create table t_base(col1 int, col2 int, col3 int); +insert into t_base values(1,2,3),(11,22,33); +create table ttt3(col int) as select * from t_base; +select * from ttt3; + \c postgres drop database b_createas; From 823c7bb89106454913b299a90d58760a346d212c Mon Sep 17 00:00:00 2001 From: yuchao Date: Fri, 11 Aug 2023 11:44:36 +0800 Subject: [PATCH 129/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dmax=5Fquery=5Fretry?= =?UTF-8?q?=5Ftimes=E5=8F=82=E6=95=B0=E6=9C=AA=E8=B5=B7=E5=88=B0=E6=8F=8F?= =?UTF-8?q?=E8=BF=B0=E6=95=88=E6=9E=9C=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/gs_guc/cluster_guc.conf | 1 - src/common/backend/utils/misc/guc.cpp | 4 ++-- src/test/regress/output/recovery_2pc_tools.source | 1 - 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/bin/gs_guc/cluster_guc.conf b/src/bin/gs_guc/cluster_guc.conf index c59516547..c426f82d4 100755 --- a/src/bin/gs_guc/cluster_guc.conf +++ b/src/bin/gs_guc/cluster_guc.conf @@ -369,7 +369,6 @@ local_syscache_threshold|int|1024,524288|kB|NULL| global_syscache_threshold|int|16384,1073741824|kB|NULL| session_statistics_memory|int|5120,1073741823|kB|NULL| session_history_memory|int|10240,1073741823|kB|NULL| -max_query_retry_times|int|0,20|NULL|NULL| max_replication_slots|int|0,1024|NULL|NULL| enable_slot_log|bool|0,0|NULL|NULL| max_changes_in_memory|int|1,2147483647|NULL|NULL| diff --git a/src/common/backend/utils/misc/guc.cpp b/src/common/backend/utils/misc/guc.cpp index d5f45e920..8092db0a7 100755 --- a/src/common/backend/utils/misc/guc.cpp +++ b/src/common/backend/utils/misc/guc.cpp @@ -360,7 +360,6 @@ const char* sync_guc_variable_namelist[] = {"work_mem", "enable_codegen_print", "codegen_cost_threshold", "codegen_strategy", - "max_query_retry_times", "convert_string_to_digit", #ifdef ENABLE_MULTIPLE_NODES "agg_redistribute_enhancement", @@ -2184,7 +2183,7 @@ static void InitConfigureNamesInt() NULL}, {{"max_query_retry_times", PGC_USERSET, - NODE_ALL, + NODE_DISTRIBUTE, CLIENT_CONN_STATEMENT, gettext_noop("Sets the maximum sql retry times."), gettext_noop("A value of 0 turns off the retry."), @@ -4787,6 +4786,7 @@ static void InitSingleNodeUnsupportGuc() u_sess->attr.attr_common.transaction_sync_timeout = 600; u_sess->attr.attr_storage.default_index_kind = DEFAULT_INDEX_KIND_GLOBAL; u_sess->attr.attr_sql.application_type = NOT_PERFECT_SHARDING_TYPE; + u_sess->attr.attr_common.max_query_retry_times = 0; /* for Double Guc Variables */ u_sess->attr.attr_sql.stream_multiple = DEFAULT_STREAM_MULTIPLE; u_sess->attr.attr_sql.max_cn_temp_file_size = 5 * 1024 * 1024; diff --git a/src/test/regress/output/recovery_2pc_tools.source b/src/test/regress/output/recovery_2pc_tools.source index 7d5db9acb..e0eeb4e50 100644 --- a/src/test/regress/output/recovery_2pc_tools.source +++ b/src/test/regress/output/recovery_2pc_tools.source @@ -479,7 +479,6 @@ select name,vartype,unit,min_val,max_val from pg_settings where name <> 'qunit_c max_pred_locks_per_transaction | integer | | 10 | 2147483647 max_prepared_transactions | integer | | 0 | 262143 max_process_memory | integer | kB | 2097152 | 2147483647 - max_query_retry_times | integer | | 0 | 20 max_recursive_times | integer | | 0 | 2147483647 max_redo_log_size | integer | kB | 163840 | 2147483647 max_replication_slots | integer | | 0 | 1024 From 370d3d83d011694dceba34c6ec142284987ca3e1 Mon Sep 17 00:00:00 2001 From: yaojun Date: Mon, 14 Aug 2023 17:18:02 +0800 Subject: [PATCH 130/304] fix: the variable is not used. --- src/common/backend/utils/error/elog.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/common/backend/utils/error/elog.cpp b/src/common/backend/utils/error/elog.cpp index 24fa62bf1..741774245 100644 --- a/src/common/backend/utils/error/elog.cpp +++ b/src/common/backend/utils/error/elog.cpp @@ -6275,7 +6275,6 @@ void getDiagnosticsInfo(List* condInfo, bool hasCondNum, List* condNum) FreeStringInfo(&buf); return; } - int currIdx = 0; DolphinErrorData* eData = (DolphinErrorData *)list_nth(errorDataArea->sqlErrorDataList, conditionNum - 1); foreach(lc, condInfo) { CondInfo *cond = (CondInfo*)lfirst(lc); From eb552a6baee8142af01be76c366bf52c5a56094a Mon Sep 17 00:00:00 2001 From: totaj Date: Tue, 15 Aug 2023 09:26:14 +0800 Subject: [PATCH 131/304] fixed 8d4fb81 from https://gitee.com/totaj/openGauss-server/pulls/3956 Must load dolphin before connect to B-database. --- src/gausskernel/process/tcop/postgres.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/gausskernel/process/tcop/postgres.cpp b/src/gausskernel/process/tcop/postgres.cpp index f05c29bf8..d99f586ba 100755 --- a/src/gausskernel/process/tcop/postgres.cpp +++ b/src/gausskernel/process/tcop/postgres.cpp @@ -7601,8 +7601,8 @@ void LoadSqlPlugin() if (!u_sess->attr.attr_sql.dolphin && !u_sess->attr.attr_common.IsInplaceUpgrade) { Oid userId = GetUserId(); if (userId != INITIAL_USER_ID) { - ereport(WARNING, (errmsg("Use the original role to load extension dolphin"))); - return; + ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("Please use the original role to connect B-compatibility database first, to load extension dolphin"))); } /* recheck and load dolphin within lock */ pthread_mutex_lock(&g_instance.loadPluginLock[DB_CMPT_B]); From 316b441e56ad1bb387cb189fa70f8b5191ff40e6 Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Tue, 15 Aug 2023 10:54:20 +0800 Subject: [PATCH 132/304] =?UTF-8?q?=E6=97=A5=E5=BF=97=E5=90=88=E4=B8=80?= =?UTF-8?q?=E8=A1=A5=E5=85=851?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/initdb/ss_initdb.cpp | 1 - .../process/postmaster/postmaster.cpp | 4 +- .../storage/access/transam/xlog.cpp | 31 ++++++------- src/gausskernel/storage/file/fd.cpp | 44 +++++++++++++++++++ .../storage/replication/libpqwalreceiver.cpp | 7 +-- .../replication/ss_cluster_replication.cpp | 9 +--- .../storage/replication/walreceiver.cpp | 13 +++--- .../storage/replication/walreceiverfuncs.cpp | 8 ++-- .../storage/replication/walsender.cpp | 6 ++- src/include/storage/smgr/fd.h | 1 + 10 files changed, 85 insertions(+), 39 deletions(-) diff --git a/src/bin/initdb/ss_initdb.cpp b/src/bin/initdb/ss_initdb.cpp index 6285d9b78..ba5fdff9a 100644 --- a/src/bin/initdb/ss_initdb.cpp +++ b/src/bin/initdb/ss_initdb.cpp @@ -375,7 +375,6 @@ void ss_createdir(const char **ss_dirs, int32 num, int32 node_id, const char *pg char *link_path = NULL; if (vglog_dir[0] != '\0' && (pg_strcasecmp(ss_dirs[i], "+pg_xlog") == 0 || - pg_strcasecmp(ss_dirs[i], "+pg_doublewrite") == 0 || pg_strcasecmp(ss_dirs[i], "+pg_notify") == 0 || pg_strcasecmp(ss_dirs[i], "+pg_snapshots") == 0 || pg_strcasecmp(ss_dirs[i], "+pg_replication") == 0)) { diff --git a/src/gausskernel/process/postmaster/postmaster.cpp b/src/gausskernel/process/postmaster/postmaster.cpp index dc04bb74e..767ed75e8 100644 --- a/src/gausskernel/process/postmaster/postmaster.cpp +++ b/src/gausskernel/process/postmaster/postmaster.cpp @@ -3076,10 +3076,12 @@ int PostmasterMain(int argc, char* argv[]) } } + /* init sharestorge(dorado) */ + ShareStorageInit(); + /* * We're ready to rock and roll... */ - ShareStorageInit(); if (ENABLE_DMS && ENABLE_REFORM) { if (!DMSWaitInitStartup()) { if (g_instance.pid_cxt.StartupPID == 0) { diff --git a/src/gausskernel/storage/access/transam/xlog.cpp b/src/gausskernel/storage/access/transam/xlog.cpp index 8864d886d..ff76a34ff 100755 --- a/src/gausskernel/storage/access/transam/xlog.cpp +++ b/src/gausskernel/storage/access/transam/xlog.cpp @@ -4200,8 +4200,14 @@ static int XLogFileOpenInternal(XLogSegNo segno, const char *xlog_dir) (uint32)((segno) / XLogSegmentsPerXLogId), (uint32)((segno) % XLogSegmentsPerXLogId)); securec_check_ss(errorno, "", ""); - fd = BasicOpenFile(path, O_RDWR | PG_BINARY | (unsigned int)get_sync_bit(u_sess->attr.attr_storage.sync_method), + if (SS_CLUSTER_DORADO_REPLICATION) { + fd = SSErgodicOpenXlogFile(segno, O_RDWR | PG_BINARY | (unsigned int)get_sync_bit(u_sess->attr.attr_storage.sync_method), + S_IRUSR | S_IWUSR); + } else { + fd = BasicOpenFile(path, O_RDWR | PG_BINARY | (unsigned int)get_sync_bit(u_sess->attr.attr_storage.sync_method), S_IRUSR | S_IWUSR); + } + if (fd < 0) { ereport(PANIC, (errcode_for_file_access(), errmsg("could not open xlog file \"%s\" (log segment %s): %m", path, XLogFileNameP(t_thrd.xlog_cxt.ThisTimeLineID, segno)))); @@ -18861,10 +18867,6 @@ void SSReadDoradoCtlInfoFile() void NormalClusterDoradoStorageInit() { - if (g_instance.attr.attr_storage.xlog_file_path == NULL || g_instance.attr.attr_storage.xlog_file_size <= 0) { - return; - } - bool found = false; void *tmpBuf = ShmemInitStruct("share storage Ctl", CalShareStorageCtlSize(), &found); g_instance.xlog_cxt.shareStorageXLogCtl = (ShareStorageXLogCtl *)TYPEALIGN(MEMORY_ALIGNED_SIZE, tmpBuf); @@ -18903,8 +18905,11 @@ void NormalClusterDoradoStorageInit() void ShareStorageInit() { - NormalClusterDoradoStorageInit(); - SSClusterDoradoStorageInit(); + if (SS_CLUSTER_DORADO_REPLICATION) { + SSClusterDoradoStorageInit(); + } else if (IS_SHARED_STORAGE_MODE) { + NormalClusterDoradoStorageInit(); + } } void ShareStorageSetBuildErrorAndExit(HaRebuildReason reason, bool setRcvDone) @@ -19185,7 +19190,7 @@ static void SSOndemandXlogCopy(XLogSegNo copySegNo, uint32 startOffset, char *co } static int SSReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, - XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *readTLI, char* xlog_path) + XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *readTLI, char* xlog_path) { /* Load reader private data */ XLogPageReadPrivate *readprivate = (XLogPageReadPrivate *)xlogreader->private_data; @@ -19233,13 +19238,6 @@ static int SSReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int XLByteToSeg(targetPagePtr, t_thrd.xlog_cxt.readSegNo); XLByteAdvance(RecPtr, reqLen); - - /* get insertHead of ctl_info file */ - if (SS_CLUSTER_DORADO_REPLICATION) { - ReadSSDoradoCtlInfoFile(); - ShareStorageXLogCtl *ctlInfo = g_instance.xlog_cxt.ssReplicationXLogCtl; - t_thrd.xlog_cxt.receivedUpto = ctlInfo->insertHead; - } retry: /* See if we need to retrieve more data */ @@ -19562,7 +19560,7 @@ static int SSReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int * know the requested record is in it. */ Assert(t_thrd.xlog_cxt.readFile != -1); - + if (t_thrd.xlog_cxt.readSource == XLOG_FROM_STREAM) { if ((targetPagePtr / XLOG_BLCKSZ) != (t_thrd.xlog_cxt.receivedUpto / XLOG_BLCKSZ)) { t_thrd.xlog_cxt.readLen = XLOG_BLCKSZ; @@ -19631,7 +19629,6 @@ static int SSReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int Assert(targetSegNo == t_thrd.xlog_cxt.readSegNo); Assert(targetPageOff == t_thrd.xlog_cxt.readOff); Assert((uint32)reqLen <= t_thrd.xlog_cxt.readLen); - *readTLI = t_thrd.xlog_cxt.curFileTLI; return t_thrd.xlog_cxt.readLen; diff --git a/src/gausskernel/storage/file/fd.cpp b/src/gausskernel/storage/file/fd.cpp index 163a1be4f..5ad4b4fc1 100755 --- a/src/gausskernel/storage/file/fd.cpp +++ b/src/gausskernel/storage/file/fd.cpp @@ -869,6 +869,50 @@ int BasicOpenFile(FileName fileName, int fileFlags, int fileMode) return -1; /* failure */ } + +/* +* When SS_CLUSTER_DORADO_REPLICATION enabled, current xlog dictionary may be not the correct dictionary, +* because all xlog dictionaries are in the same LUN, we need loop over other dictionaries. +*/ +int SSErgodicOpenXlogFile(XLogSegNo segno, int fileFlags, int fileMode) +{ + char xlog_file_name[MAXPGPATH]; + char xlog_file_full_path[MAXPGPATH]; + char *dssdir = g_instance.attr.attr_storage.dss_attr.ss_dss_vg_name; + DIR* dir; + int fd; + struct dirent *entry; + errno_t errorno = EOK; + + errorno = snprintf_s(xlog_file_name, MAXPGPATH, MAXPGPATH - 1, "%08X%08X%08X", t_thrd.xlog_cxt.ThisTimeLineID, + (uint32)((segno) / XLogSegmentsPerXLogId), (uint32)((segno) % XLogSegmentsPerXLogId)); + securec_check_ss(errorno, "", ""); + + dir = opendir(dssdir); + if (dir == NULL) { + ereport(PANIC, (errcode_for_file_access(), errmsg("Error opening dssdir %s", dssdir))); + } + + while ((entry = readdir(dir)) != NULL) { + if (strncmp(entry->d_name, "pg_xlog", strlen("pg_xlog")) == 0) { + errorno = snprintf_s(xlog_file_full_path, MAXPGPATH, MAXPGPATH - 1, "%s/%s/%s", dssdir, entry->d_name, xlog_file_name); + securec_check_ss(errorno, "", ""); + + fd = BasicOpenFile(xlog_file_full_path, fileFlags, fileMode); + if (fd >= 0) { + return fd; + } + } + } + + if (fd < 0) { + ereport(PANIC, (errcode_for_file_access(), errmsg("could not open xlog file \"%s\" (log segment %s): %m", xlog_file_name, + XLogFileNameP(t_thrd.xlog_cxt.ThisTimeLineID, segno)))); + } + + return fd; +} + #if defined(FDDEBUG) static void _dump_lru(void) diff --git a/src/gausskernel/storage/replication/libpqwalreceiver.cpp b/src/gausskernel/storage/replication/libpqwalreceiver.cpp index d211663fc..ebf0e1cde 100755 --- a/src/gausskernel/storage/replication/libpqwalreceiver.cpp +++ b/src/gausskernel/storage/replication/libpqwalreceiver.cpp @@ -524,7 +524,7 @@ ServerMode IdentifyRemoteMode() } } - if (SS_CLUSTER_DORADO_REPLICATION && CheckSSRemoteServerMode(remoteMode, res)) { + if (SS_CLUSTER_DORADO_REPLICATION && !CheckSSRemoteServerMode(remoteMode, res)) { return UNKNOWN_MODE; } @@ -577,7 +577,7 @@ static int32 IdentifyRemoteVersion() (errcode(ERRCODE_INVALID_STATUS), errmsg("could not get the local protocal version, make sure the PG_PROTOCOL_VERSION is defined"))); } - if (!IS_SHARED_STORAGE_STANDBY_CLUSTER_STANDBY_MODE) { + if (!IS_SHARED_STORAGE_STANDBY_CLUSTER_STANDBY_MODE && !IS_SS_REPLICATION_MAIN_STANBY_NODE) { if (walrcv->conn_target != REPCONNTARGET_DUMMYSTANDBY && (localTerm == 0 || localTerm > remoteTerm) && !AM_HADR_WAL_RECEIVER) { PQclear(res); @@ -699,7 +699,8 @@ bool libpqrcv_connect(char *conninfo, XLogRecPtr *startpoint, char *slotname, in u_sess->attr.attr_storage.wal_receiver_connect_timeout, username, passwd); rc = memset_s(passwd, MAXPGPATH, 0, MAXPGPATH); securec_check(rc, "\0", "\0"); - } else if (IS_SHARED_STORAGE_STANDBY_CLUSTER_STANDBY_MODE) { + } else if (IS_SHARED_STORAGE_STANDBY_CLUSTER_STANDBY_MODE || + IS_SS_REPLICATION_MAIN_STANBY_NODE) { nRet = snprintf_s(conninfoRepl, sizeof(conninfoRepl), sizeof(conninfoRepl) - 1, "%s dbname=postgres replication=standby_cluster " "fallback_application_name=%s_hass " diff --git a/src/gausskernel/storage/replication/ss_cluster_replication.cpp b/src/gausskernel/storage/replication/ss_cluster_replication.cpp index 3283018b9..7014e27b3 100644 --- a/src/gausskernel/storage/replication/ss_cluster_replication.cpp +++ b/src/gausskernel/storage/replication/ss_cluster_replication.cpp @@ -151,17 +151,10 @@ void InitSSDoradoCtlInfoFile() void SSClusterDoradoStorageInit() { - if (!SS_CLUSTER_DORADO_REPLICATION) { - return; - } - bool found = false; g_instance.xlog_cxt.ssReplicationXLogCtl = (ShareStorageXLogCtl*)ShmemInitStruct("SS Replication Xlog Ctl", sizeof(ShareStorageXLogCtl), &found); InitSSDoradoCtlInfoFile(); - - if (!IsInitdb && ENABLE_DMS) { - g_instance.dms_cxt.SSRecoveryInfo.dorado_sharestorage_inited = true; - } + g_instance.dms_cxt.SSRecoveryInfo.dorado_sharestorage_inited = true; } void UpdateSSDoradoCtlInfoAndSync() diff --git a/src/gausskernel/storage/replication/walreceiver.cpp b/src/gausskernel/storage/replication/walreceiver.cpp index 3d8c4b770..470373c8d 100755 --- a/src/gausskernel/storage/replication/walreceiver.cpp +++ b/src/gausskernel/storage/replication/walreceiver.cpp @@ -387,7 +387,8 @@ void WalRcvrProcessData(TimestampTz *last_recv_timestamp, bool *ping_sent) } } - if (!WalRcvWriterInProgress()) + /* walrec in dorado replication mode not need walrecwrite */ + if (!IS_SS_REPLICATION_MAIN_STANBY_NODE && !WalRcvWriterInProgress()) ereport(FATAL, (errmsg("terminating walreceiver process due to the death of walrcvwriter"))); #ifndef ENABLE_MULTIPLE_NODES /* For Paxos, receive wal should be done by send log callback function */ @@ -578,7 +579,7 @@ void WalReceiverMain(void) int walreplindex = hashmdata->current_repl; SpinLockRelease(&hashmdata->mutex); - if (!IS_SHARED_STORAGE_STANDBY_CLUSTER_STANDBY_MODE) { + if (!IS_SHARED_STORAGE_STANDBY_CLUSTER_STANDBY_MODE && !IS_SS_REPLICATION_MAIN_STANBY_NODE) { replConnInfo = t_thrd.postmaster_cxt.ReplConnArray[walreplindex]; } else if (walreplindex >= MAX_REPLNODE_NUM) { replConnInfo = t_thrd.postmaster_cxt.CrossClusterReplConnArray[walreplindex - MAX_REPLNODE_NUM]; @@ -1132,9 +1133,11 @@ static void XLogWalRcvProcessMsg(unsigned char type, char *buf, Size len) case 'm': /* config file */ { #ifndef ENABLE_MULTIPLE_NODES - if (IS_SHARED_STORAGE_STANDBY_CLUSTER_STANDBY_MODE || AM_HADR_WAL_RECEIVER) { + if (IS_SHARED_STORAGE_STANDBY_CLUSTER_STANDBY_MODE || AM_HADR_WAL_RECEIVER + || SS_CLUSTER_DORADO_REPLICATION) { #else - if (IS_SHARED_STORAGE_STANDBY_CLUSTER_STANDBY_MODE || AM_HADR_WAL_RECEIVER || AM_HADR_CN_WAL_RECEIVER) { + if (IS_SHARED_STORAGE_STANDBY_CLUSTER_STANDBY_MODE || AM_HADR_WAL_RECEIVER + || SS_CLUSTER_DORADO_REPLICATION || AM_HADR_CN_WAL_RECEIVER) { #endif break; } @@ -1613,7 +1616,7 @@ void XLogWalRcvSendReply(bool force, bool requestReply) t_thrd.walreceiver_cxt.reply_message->replyRequested = requestReply; SpinLockAcquire(&hashmdata->mutex); - if (!IS_SHARED_STORAGE_STANDBY_CLUSTER_STANDBY_MODE) + if (!IS_SHARED_STORAGE_STANDBY_CLUSTER_STANDBY_MODE && !IS_SS_REPLICATION_MAIN_STANBY_NODE) t_thrd.walreceiver_cxt.reply_message->peer_role = hashmdata->current_mode; else t_thrd.walreceiver_cxt.reply_message->peer_role = STANDBY_CLUSTER_MODE; diff --git a/src/gausskernel/storage/replication/walreceiverfuncs.cpp b/src/gausskernel/storage/replication/walreceiverfuncs.cpp index 9e47555a5..b56cc0c0a 100755 --- a/src/gausskernel/storage/replication/walreceiverfuncs.cpp +++ b/src/gausskernel/storage/replication/walreceiverfuncs.cpp @@ -36,6 +36,7 @@ #include "replication/replicainternal.h" #include "replication/walreceiver.h" #include "replication/slot.h" +#include "replication/ss_cluster_replication.h" #include "replication/dcf_replication.h" #include "replication/walsender_private.h" #include "storage/pmsignal.h" @@ -259,7 +260,7 @@ static void SetWalRcvConninfo(ReplConnTarget conn_target) SpinLockRelease(&walrcv->mutex); ereport(LOG, (errmsg("wal receiver try to connect to %s index %d .", walrcv->conninfo, useIndex))); SpinLockAcquire(&hashmdata->mutex); - if (!IS_SHARED_STORAGE_STANDBY_CLUSTER_STANDBY_MODE) + if (!IS_SHARED_STORAGE_STANDBY_CLUSTER_STANDBY_MODE && !IS_SS_REPLICATION_MAIN_STANBY_NODE) hashmdata->current_repl = useIndex; else hashmdata->current_repl = MAX_REPLNODE_NUM + useIndex; @@ -416,7 +417,8 @@ static void set_rcv_slot_name(const char *slotname) SpinLockAcquire(&hashmdata->mutex); replIdx = hashmdata->current_repl; SpinLockRelease(&hashmdata->mutex); - if (IS_SHARED_STORAGE_STANDBY_CLUSTER_STANDBY_MODE && replIdx >= MAX_REPLNODE_NUM) { + if ((IS_SHARED_STORAGE_STANDBY_CLUSTER_STANDBY_MODE || IS_SS_REPLICATION_MAIN_STANBY_NODE) + && replIdx >= MAX_REPLNODE_NUM) { replIdx = replIdx - MAX_REPLNODE_NUM; } conninfo = GetRepConnArray(&replIdx); @@ -891,7 +893,7 @@ ReplConnInfo *GetRepConnArray(int *cur_idx) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid replication node index:%d", *cur_idx))); } - if (!IS_SHARED_STORAGE_STANDBY_CLUSTER_STANDBY_MODE) + if (!IS_SHARED_STORAGE_STANDBY_CLUSTER_STANDBY_MODE && !IS_SS_REPLICATION_MAIN_STANBY_NODE) replConnInfoArray = &t_thrd.postmaster_cxt.ReplConnArray[0]; else replConnInfoArray = &t_thrd.postmaster_cxt.CrossClusterReplConnArray[0]; diff --git a/src/gausskernel/storage/replication/walsender.cpp b/src/gausskernel/storage/replication/walsender.cpp index 0ec90fa7f..178e8b97c 100755 --- a/src/gausskernel/storage/replication/walsender.cpp +++ b/src/gausskernel/storage/replication/walsender.cpp @@ -3558,7 +3558,11 @@ static void LogCtrlCalculateCurrentRPO(StandbyReplyMessage *reply) if (AM_WAL_HADR_CN_SENDER) { flushPtr = GetFlushRecPtr(); } else if (AM_WAL_SHARE_STORE_SENDER) { - flushPtr = g_instance.xlog_cxt.shareStorageXLogCtl->insertHead; + if (SS_CLUSTER_DORADO_REPLICATION) { + flushPtr = g_instance.xlog_cxt.ssReplicationXLogCtl->insertHead; + } else { + flushPtr = g_instance.xlog_cxt.shareStorageXLogCtl->insertHead; + } } else { got_recptr = SyncRepGetSyncRecPtr(&receivePtr, &writePtr, &flushPtr, &replayPtr, &amSync, false); if (got_recptr != true) { diff --git a/src/include/storage/smgr/fd.h b/src/include/storage/smgr/fd.h index f6a6ed651..99e019714 100755 --- a/src/include/storage/smgr/fd.h +++ b/src/include/storage/smgr/fd.h @@ -131,6 +131,7 @@ extern int OpenTransientFile(FileName fileName, int fileFlags, int fileMode); extern int CloseTransientFile(int fd); /* If you've really really gotta have a plain kernel FD, use this */ extern int BasicOpenFile(FileName fileName, int fileFlags, int fileMode); +extern int SSErgodicOpenXlogFile(XLogSegNo segno, int fileFlags, int fileMode); /* Miscellaneous support routines */ extern void InitFileAccess(void); From 914b0620c92789099b3099f44f6907ca7a3c3ea9 Mon Sep 17 00:00:00 2001 From: yuchao Date: Tue, 15 Aug 2023 17:14:26 +0800 Subject: [PATCH 133/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=BE=80=E6=97=B6?= =?UTF-8?q?=E9=97=B4=E7=B1=BB=E5=9E=8B=E6=8F=92=E5=85=A5=E6=95=B0=E5=AD=97?= =?UTF-8?q?ignore=E5=A4=B1=E6=95=88=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/runtime/executor/execQual.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/gausskernel/runtime/executor/execQual.cpp b/src/gausskernel/runtime/executor/execQual.cpp index 69a49a6d8..4965f4a03 100644 --- a/src/gausskernel/runtime/executor/execQual.cpp +++ b/src/gausskernel/runtime/executor/execQual.cpp @@ -2732,7 +2732,8 @@ static Datum ExecMakeFunctionResultNoSets( } if (econtext) { - fcinfo->can_ignore = econtext->can_ignore; + fcinfo->can_ignore = econtext->can_ignore || (econtext->ecxt_estate && econtext->ecxt_estate->es_plannedstmt && + econtext->ecxt_estate->es_plannedstmt->hasIgnore); } /* From cf1d6f0b67cbb47ac8a7ce22622d09f5af56eff7 Mon Sep 17 00:00:00 2001 From: zhangao_za <18829237393@163.com> Date: Sat, 5 Aug 2023 18:33:20 +0800 Subject: [PATCH 134/304] =?UTF-8?q?dorado=E5=8F=8C=E9=9B=86=E7=BE=A4?= =?UTF-8?q?=EF=BC=8C=E5=AE=9E=E7=8E=B0=E9=9B=86=E7=BE=A4=E9=97=B4failover?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/gs_guc/cluster_guc.conf | 1 - src/bin/pg_controldata/pg_controldata.cpp | 13 ++++ src/bin/pg_ctl/pg_ctl.cpp | 2 +- src/common/backend/utils/misc/guc.cpp | 14 +--- .../ddes/adapter/ss_dms_recovery.cpp | 1 + .../process/postmaster/postmaster.cpp | 35 +++++++++- .../storage/access/transam/xlog.cpp | 69 ++++++++++++++++++- src/include/ddes/dms/ss_dms_recovery.h | 1 + src/test/ha/standby_env.sh | 36 ++++++++-- src/test/ha/util.sh | 33 +++------ .../regress/output/recovery_2pc_tools.source | 1 - .../common_function.sh | 22 +++--- 12 files changed, 169 insertions(+), 59 deletions(-) diff --git a/src/bin/gs_guc/cluster_guc.conf b/src/bin/gs_guc/cluster_guc.conf index ae2c84f92..11841fdb3 100755 --- a/src/bin/gs_guc/cluster_guc.conf +++ b/src/bin/gs_guc/cluster_guc.conf @@ -701,7 +701,6 @@ time_to_target_rpo|int|0,3600|NULL|NULL| disable_memory_protect|bool|0,0|NULL|NULL| segment_buffers|int|16,1073741823|kB|NULL| undo_zone_count|int|0,1048576|NULL|NULL| -cluster_run_mode|enum|cluster_primary,cluster_standby|NULL|NULL| stream_cluster_run_mode|enum|cluster_primary,cluster_standby|NULL|NULL| xlog_file_size|int64|1048576,576460752303423487|B|The value must be an integer multiple of 16777216(16M)| xlog_file_path|string|0,0|NULL|NULL| diff --git a/src/bin/pg_controldata/pg_controldata.cpp b/src/bin/pg_controldata/pg_controldata.cpp index f8fe52c25..67f6dc2f8 100644 --- a/src/bin/pg_controldata/pg_controldata.cpp +++ b/src/bin/pg_controldata/pg_controldata.cpp @@ -98,6 +98,18 @@ static const char* SSClusterState(SSGlobalClusterState state) { return _("unrecognized status code"); } +static const char* SSClusterRunMode(ClusterRunMode run_mode) { + switch (run_mode) { + case RUN_MODE_PRIMARY: + return _("primary cluster"); + case RUN_MODE_STANDBY: + return _("standby cluster"); + default: + break; + } + return _("unrecognized cluster run mode"); +} + static const char* wal_level_str(WalLevel wal_level) { switch (wal_level) { @@ -263,6 +275,7 @@ static void display_last_page(ss_reformer_ctrl_t reformerCtrl, int last_page_id) printf(_("Primary instance ID: %d\n"), reformerCtrl.primaryInstId); printf(_("Recovery instance ID: %d\n"), reformerCtrl.recoveryInstId); printf(_("Cluster status: %s\n"), SSClusterState(reformerCtrl.clusterStatus)); + printf(_("Cluster run mode: %s\n"), SSClusterRunMode(reformerCtrl.clusterRunMode)); } int main(int argc, char* argv[]) diff --git a/src/bin/pg_ctl/pg_ctl.cpp b/src/bin/pg_ctl/pg_ctl.cpp index 1a2995480..3984fef1a 100755 --- a/src/bin/pg_ctl/pg_ctl.cpp +++ b/src/bin/pg_ctl/pg_ctl.cpp @@ -2203,7 +2203,7 @@ static void do_failover(uint32 term) exit(1); } /* failover executed only in standby server */ - else if (run_mode != STANDBY_MODE && run_mode != CASCADE_STANDBY_MODE) { + else if (run_mode != STANDBY_MODE && run_mode != CASCADE_STANDBY_MODE && run_mode != MAIN_STANDBY_MODE) { pg_log(PG_WARNING, _(" cannot failover server; " "server is not in standby or cascade standby mode\n")); diff --git a/src/common/backend/utils/misc/guc.cpp b/src/common/backend/utils/misc/guc.cpp index 8092db0a7..4cee8fb94 100755 --- a/src/common/backend/utils/misc/guc.cpp +++ b/src/common/backend/utils/misc/guc.cpp @@ -852,7 +852,7 @@ static const struct config_enum_entry sync_config_strategy_options[] = { #endif static const struct config_enum_entry cluster_run_mode_options[] = { - {"cluster_primary", RUN_MODE_PRIMARY, false}, + {"cluster_primary", RUN_MODE_PRIMARY, false}, {"cluster_standby", RUN_MODE_STANDBY, false}, {NULL, 0, false}}; @@ -4409,18 +4409,6 @@ static void InitConfigureNamesEnum() NULL, NULL, NULL}, - {{"cluster_run_mode", - PGC_POSTMASTER, - NODE_SINGLENODE, - PRESET_OPTIONS, - gettext_noop("Sets the type of shared storage cluster."), - NULL}, - &g_instance.attr.attr_common.cluster_run_mode, - RUN_MODE_PRIMARY, - cluster_run_mode_options, - NULL, - NULL, - NULL}, {{"stream_cluster_run_mode", PGC_POSTMASTER, NODE_ALL, diff --git a/src/gausskernel/ddes/adapter/ss_dms_recovery.cpp b/src/gausskernel/ddes/adapter/ss_dms_recovery.cpp index b8a5a2c62..92c38eb6d 100644 --- a/src/gausskernel/ddes/adapter/ss_dms_recovery.cpp +++ b/src/gausskernel/ddes/adapter/ss_dms_recovery.cpp @@ -194,6 +194,7 @@ void SSInitReformerControlPages(void) g_instance.dms_cxt.SSReformerControl.recoveryInstId = INVALID_INSTANCEID; g_instance.dms_cxt.SSReformerControl.version = REFORM_CTRL_VERSION; g_instance.dms_cxt.SSReformerControl.clusterStatus = CLUSTER_NORMAL; + g_instance.dms_cxt.SSReformerControl.clusterRunMode = RUN_MODE_PRIMARY; (void)printf("[SS] Current node:%d initdb first, will become PRIMARY for first-time SS cluster startup.\n", SS_MY_INST_ID); diff --git a/src/gausskernel/process/postmaster/postmaster.cpp b/src/gausskernel/process/postmaster/postmaster.cpp index dc04bb74e..dd44c6c12 100644 --- a/src/gausskernel/process/postmaster/postmaster.cpp +++ b/src/gausskernel/process/postmaster/postmaster.cpp @@ -1896,7 +1896,7 @@ int PostmasterMain(int argc, char* argv[]) optCtxt.opterr = 1; - check_short_optOfVoid("A:B:bc:C:D:d:EeFf:h:ijk:lM:N:nOo:Pp:Rr:S:sTt:u:W:g:X:-:", argc, argv); + check_short_optOfVoid("A:B:bc:C:D:d:EeFf:h:ijk:lM:N:nOo:Pp:Rr:S:sTt:u:W:g:X:z:-:", argc, argv); /* * Parse command-line options. CAUTION: keep this in sync with @@ -1904,7 +1904,7 @@ int PostmasterMain(int argc, char* argv[]) * common help() function in main/main.c. */ initOptParseContext(&optCtxt); - while ((opt = getopt_r(argc, argv, "A:B:bc:C:D:d:EeFf:h:ijk:lM:N:nOo:Pp:Rr:S:sTt:u:W:g:X:-:", &optCtxt)) != -1) { + while ((opt = getopt_r(argc, argv, "A:B:bc:C:D:d:EeFf:h:ijk:lM:N:nOo:Pp:Rr:S:sTt:u:W:g:X:z:-:", &optCtxt)) != -1) { switch (opt) { case 'A': SetConfigOption("debug_assertions", optCtxt.optarg, PGC_POSTMASTER, PGC_S_ARGV); @@ -2101,6 +2101,18 @@ int PostmasterMain(int argc, char* argv[]) ereport(LOG, (errmsg("Set stop barrierID %s", g_instance.csn_barrier_cxt.stopBarrierId))); } break; + case 'z': + if (0 == strncmp(optCtxt.optarg, "cluster_primary", strlen("cluster_primary")) && + '\0' == optCtxt.optarg[strlen("cluster_primary")]) { + g_instance.attr.attr_common.cluster_run_mode = RUN_MODE_PRIMARY; + } else if (0 == strncmp(optCtxt.optarg, "cluster_standby", strlen("cluster_standby")) && + '\0' == optCtxt.optarg[strlen("cluster_standby")]) { + g_instance.attr.attr_common.cluster_run_mode = RUN_MODE_STANDBY; + } else { + ereport(FATAL, (errmsg("the options of -z is not recognized"))); + } + g_instance.dms_cxt.SSReformerControl.clusterRunMode = (ClusterRunMode)g_instance.attr.attr_common.cluster_run_mode; + break; case 'c': case '-': { char* name = NULL; @@ -9961,6 +9973,23 @@ static void sigusr1_handler(SIGNAL_ARGS) (errmsg("set gaussdb state file: db state(PROMOTING_STATE), server mode(%s)", wal_get_role_string(get_cur_mode())))); + /* + * update cluster_run_mode from pg_control file, + * in case failover has been performed between two dorado cluster. + */ + if (ENABLE_DMS && g_instance.attr.attr_storage.xlog_file_path != 0) { + g_instance.attr.attr_common.cluster_run_mode = g_instance.dms_cxt.SSReformerControl.clusterRunMode; + } + if (ENABLE_DMS && DORADO_STANDBY_CLUSTER_MAINSTANDBY_NODE) { + ereport(LOG, + (errmsg("Failover between two dorado cluster start, change current run mode to primary_cluster"))); + g_instance.attr.attr_common.cluster_run_mode = RUN_MODE_PRIMARY; + g_instance.dms_cxt.SSReformerControl.clusterRunMode = RUN_MODE_PRIMARY; + SSSaveReformerCtrl(); + t_thrd.xlog_cxt.server_mode = PRIMARY_MODE; + SetHaShmemData(); + } + /* promote cascade standby */ if (IsCascadeStandby()) { t_thrd.xlog_cxt.is_cascade_standby = false; @@ -10120,6 +10149,8 @@ static void sigusr1_handler(SIGNAL_ARGS) if (ENABLE_DMS && (mode = CheckSwitchoverSignal())) { SSReadControlFile(REFORM_CTRL_PAGE); if (SS_NORMAL_STANDBY && pmState == PM_RUN && !SS_STANDBY_ONDEMAND_RECOVERY) { + /* update cluster_run_mode in case failover has been performed between two dorado cluster. */ + g_instance.attr.attr_common.cluster_run_mode = g_instance.dms_cxt.SSReformerControl.clusterRunMode; SSDoSwitchover(); } else { ereport(LOG, (errmsg("Current mode is not NORMAL STANDBY, SS switchover command ignored."))); diff --git a/src/gausskernel/storage/access/transam/xlog.cpp b/src/gausskernel/storage/access/transam/xlog.cpp index 8864d886d..2ca67e18f 100755 --- a/src/gausskernel/storage/access/transam/xlog.cpp +++ b/src/gausskernel/storage/access/transam/xlog.cpp @@ -9579,6 +9579,11 @@ void StartupXLOG(void) } } + if (SS_PRIMARY_MODE || SS_STANDBY_CLUSTER_MAIN_STANDBY) { + g_instance.dms_cxt.SSReformerControl.clusterRunMode = (ClusterRunMode)g_instance.attr.attr_common.cluster_run_mode; + SSSaveReformerCtrl(); + } + ReadRemainSegsFile(); /* Determine whether it is currently in the switchover of streaming disaster recovery */ checkHadrInSwitchover(); @@ -19359,6 +19364,30 @@ static int SSReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int t_thrd.xlog_cxt.RedoDone = IsRedoDonePromoting(); pg_memory_barrier(); + if (DORADO_STANDBY_CLUSTER_MAINSTANDBY_NODE && WalRcvIsDone() && CheckForFailoverTrigger()) { + t_thrd.xlog_cxt.receivedUpto = GetWalRcvWriteRecPtr(NULL); + if (XLByteLT(RecPtr, t_thrd.xlog_cxt.receivedUpto)) { + /* wait xlog redo done */ + continue; + } + + ProcTxnWorkLoad(true); + /* use volatile pointer to prevent code rearrangement */ + volatile WalRcvData *walrcv = t_thrd.walreceiverfuncs_cxt.WalRcv; + SpinLockAcquire(&walrcv->mutex); + walrcv->dummyStandbyConnectFailed = false; + SpinLockRelease(&walrcv->mutex); + + ereport(LOG, (errmsg("RecPtr(%X/%X),receivedUpto(%X/%X)", (uint32)(RecPtr >> 32), + (uint32)RecPtr, (uint32)(t_thrd.xlog_cxt.receivedUpto >> 32), + (uint32)t_thrd.xlog_cxt.receivedUpto))); + + ShutdownWalRcv(); + ShutdownDataRcv(); + + goto triggered; + } + if (IS_SHARED_STORAGE_MODE || SS_CLUSTER_DORADO_REPLICATION) { uint32 disableConnectionNode = pg_atomic_read_u32(&g_instance.comm_cxt.localinfo_cxt.need_disable_connection_node); @@ -19486,7 +19515,9 @@ static int SSReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int */ load_server_mode(); - if (DORADO_STANDBY_CLUSTER_MAINSTANDBY_NODE || IS_SS_REPLICATION_MAIN_STANBY_NODE) { + if (DORADO_STANDBY_CLUSTER_MAINSTANDBY_NODE && CheckForFailoverTrigger()) { + goto triggered; + } else if (DORADO_STANDBY_CLUSTER_MAINSTANDBY_NODE || IS_SS_REPLICATION_MAIN_STANBY_NODE) { ProcTxnWorkLoad(false); /* use volatile pointer to prevent code rearrangement */ volatile WalRcvData *walrcv = t_thrd.walreceiverfuncs_cxt.WalRcv; @@ -19524,6 +19555,32 @@ static int SSReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int */ t_thrd.xlog_cxt.failedSources |= sources; + if (DORADO_STANDBY_CLUSTER_MAINSTANDBY_NODE && CheckForFailoverTrigger()) { + XLogRecPtr receivedUpto = GetWalRcvWriteRecPtr(NULL); + XLogRecPtr EndRecPtrTemp = t_thrd.xlog_cxt.EndRecPtr; + XLByteAdvance(EndRecPtrTemp, SizeOfXLogRecord); + if (XLByteLT(EndRecPtrTemp, receivedUpto) && !FORCE_FINISH_ENABLED && + t_thrd.xlog_cxt.currentRetryTimes++ < g_retryTimes) { + ereport(WARNING, (errmsg("there are some received xlog have not been redo " + "the tail of last redo lsn:%X/%X, received lsn:%X/%X, retry %d times", + (uint32)(EndRecPtrTemp >> 32), (uint32)EndRecPtrTemp, + (uint32)(receivedUpto >> 32), (uint32)receivedUpto, + t_thrd.xlog_cxt.currentRetryTimes))); + return -1; + } + ereport(LOG, + (errmsg("read record failed when promoting, current lsn (%X/%X), received lsn(%X/%X)," + "sources[%u], failedSources[%u], readSource[%u], readFile[%d], readId[%u]," + "readSeg[%u], readOff[%u], readLen[%u]", + (uint32)(RecPtr >> 32), (uint32)RecPtr, + (uint32)(t_thrd.xlog_cxt.receivedUpto >> 32), + (uint32)t_thrd.xlog_cxt.receivedUpto, sources, t_thrd.xlog_cxt.failedSources, + t_thrd.xlog_cxt.readSource, t_thrd.xlog_cxt.readFile, + (uint32)(t_thrd.xlog_cxt.readSegNo >> 32), (uint32)t_thrd.xlog_cxt.readSegNo, + t_thrd.xlog_cxt.readOff, t_thrd.xlog_cxt.readLen))); + goto triggered; + } + } /* @@ -19646,6 +19703,16 @@ static int SSReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int t_thrd.xlog_cxt.readLen = 0; t_thrd.xlog_cxt.readSource = 0; + return -1; +triggered: + if (t_thrd.xlog_cxt.readFile >= 0) { + close(t_thrd.xlog_cxt.readFile); + } + t_thrd.xlog_cxt.readFile = -1; + t_thrd.xlog_cxt.readLen = 0; + t_thrd.xlog_cxt.readSource = 0; + t_thrd.xlog_cxt.recoveryTriggered = true; + return -1; } diff --git a/src/include/ddes/dms/ss_dms_recovery.h b/src/include/ddes/dms/ss_dms_recovery.h index 0c5dc6547..308ac92d8 100644 --- a/src/include/ddes/dms/ss_dms_recovery.h +++ b/src/include/ddes/dms/ss_dms_recovery.h @@ -55,6 +55,7 @@ typedef struct st_reformer_ctrl { int primaryInstId; int recoveryInstId; SSGlobalClusterState clusterStatus; + ClusterRunMode clusterRunMode; pg_crc32c crc; } ss_reformer_ctrl_t; diff --git a/src/test/ha/standby_env.sh b/src/test/ha/standby_env.sh index 6ff765438..c9e37b85e 100644 --- a/src/test/ha/standby_env.sh +++ b/src/test/ha/standby_env.sh @@ -488,8 +488,12 @@ function check_multi_standby_startup() } function start_standby() { +cluster_mode_param="" +if [-n "$1" ]; then + cluster_mode_param=$1 +fi echo "start standby $standby_data_dir" -$bin_dir/gaussdb --single_node -M standby -p $dn1_standby_port -D $standby_data_dir > ./results/gaussdb.log 2>&1 & +$bin_dir/gaussdb --single_node -M standby -p $dn1_standby_port -D $standby_data_dir $cluster_mode_param > ./results/gaussdb.log 2>&1 & check_standby_startup } @@ -502,15 +506,23 @@ sleep 10 function start_standby2() { + cluster_mode_param="" + if [-n "$1" ]; then + cluster_mode_param=$1 + fi echo "start standby2 $standby2_data_dir" - $bin_dir/gaussdb --single_node -M standby -p $standby2_port -D $standby2_data_dir > ./results/gaussdb.log 2>&1 & + $bin_dir/gaussdb --single_node -M standby -p $standby2_port -D $standby2_data_dir $cluster_mode_param > ./results/gaussdb.log 2>&1 & sleep 10 } function start_standby3() { + cluster_mode_param="" + if [-n "$1" ]; then + cluster_mode_param=$1 + fi echo "start standby3 $standby3_data_dir" - $bin_dir/gaussdb --single_node -M standby -p $standby3_port -D $standby3_data_dir > ./results/gaussdb.log 2>&1 & + $bin_dir/gaussdb --single_node -M standby -p $standby3_port -D $standby3_data_dir $cluster_mode_param > ./results/gaussdb.log 2>&1 & sleep 10 } @@ -523,15 +535,23 @@ function start_standby4() function start_primary_as_primary() { +cluster_mode_param="" +if [-n "$1" ]; then + cluster_mode_param=$1 +fi echo "start primary $primary_data_dir as primary" -$bin_dir/gaussdb --single_node -M primary -p $dn1_primary_port -D $primary_data_dir > ./results/gaussdb.log 2>&1 & +$bin_dir/gaussdb --single_node -M primary -p $dn1_primary_port -D $primary_data_dir $cluster_mode_param > ./results/gaussdb.log 2>&1 & check_primary_startup } function start_primary_as_standby() { +cluster_mode_param="" +if [-n "$1" ]; then + cluster_mode_param=$1 +fi echo "start primary $primary_data_dir as standby" -$bin_dir/gaussdb --single_node -M standby -p $dn1_primary_port -D $primary_data_dir > ./results/gaussdb.log 2>&1 & +$bin_dir/gaussdb --single_node -M standby -p $dn1_primary_port -D $primary_data_dir $cluster_mode_param > ./results/gaussdb.log 2>&1 & check_primary_startup } @@ -544,8 +564,12 @@ check_standby_startup function start_standby2_as_primary() { +cluster_mode_param="" +if [-n "$1" ]; then + cluster_mode_param=$1 +fi echo "start standby $standby2_data_dir as primary" -$bin_dir/gaussdb --single_node -M primary -p $standby2_port -D $standby2_data_dir > ./results/gaussdb.log 2>&1 & +$bin_dir/gaussdb --single_node -M primary -p $standby2_port -D $standby2_data_dir $cluster_mode_param > ./results/gaussdb.log 2>&1 & check_standby2_startup } diff --git a/src/test/ha/util.sh b/src/test/ha/util.sh index 3f832dbd7..7f294aed1 100644 --- a/src/test/ha/util.sh +++ b/src/test/ha/util.sh @@ -194,42 +194,27 @@ function kill_standby_cluster() { function start_primary_cluster() { cluster_dns=($primary_data_dir $standby_data_dir) - for element in ${cluster_dns[@]} - do - gs_guc set -Z datanode -D $element -c "cluster_run_mode=cluster_primary" - done - start_primary_as_primary - start_standby + start_primary_as_primary "-z cluster_primary" + start_standby "-z cluster_primary" } function start_standby_cluster() { cluster_dns=($standby2_data_dir $standby3_data_dir) - for element in ${cluster_dns[@]} - do - gs_guc set -Z datanode -D $element -c "cluster_run_mode=cluster_standby" - done - start_standby2 - start_standby3 + + start_standby2 "-z cluster_standby" + start_standby3 "-z cluster_standby" } function start_primary_cluster_as_standby() { cluster_dns=($primary_data_dir $standby_data_dir) - for element in ${cluster_dns[@]} - do - gs_guc set -Z datanode -D $element -c "cluster_run_mode=cluster_standby" - done - start_primary_as_standby - start_standby + start_primary_as_standby "-z cluster_standby" + start_standby "-z cluster_standby" } function start_standby_cluster_as_primary() { cluster_dns=($standby2_data_dir $standby3_data_dir) - for element in ${cluster_dns[@]} - do - gs_guc set -Z datanode -D $element -c "cluster_run_mode=cluster_primary" - done - start_standby2_as_primary - start_standby3 + start_standby2_as_primary "-z cluster_primary" + start_standby3 "-z cluster_primary" } function stop_primary_cluster() { diff --git a/src/test/regress/output/recovery_2pc_tools.source b/src/test/regress/output/recovery_2pc_tools.source index fc237cfbd..991902d04 100644 --- a/src/test/regress/output/recovery_2pc_tools.source +++ b/src/test/regress/output/recovery_2pc_tools.source @@ -110,7 +110,6 @@ select name,vartype,unit,min_val,max_val from pg_settings where name <> 'qunit_c checkpoint_warning | integer | s | 0 | 2147483647 client_encoding | string | | | client_min_messages | enum | | | - cluster_run_mode | enum | | | cn_send_buffer_size | integer | kB | 8 | 128 codegen_cost_threshold | integer | | 0 | 2147483647 codegen_strategy | enum | | | diff --git a/src/test/ss/dual_cluster_single_shared_storage/common_function.sh b/src/test/ss/dual_cluster_single_shared_storage/common_function.sh index 489f08942..42d75e5e7 100644 --- a/src/test/ss/dual_cluster_single_shared_storage/common_function.sh +++ b/src/test/ss/dual_cluster_single_shared_storage/common_function.sh @@ -58,8 +58,9 @@ function check_dn_startup() function start_dn() { data_dir=$1 - echo "start $data_dir" - nohup $BIN_PATH/gaussdb -D $data_dir & 2>&1 & + run_mode=$2 + echo "start $data_dir, exec gaussdb -D $data_dir $run_mode" + nohup $BIN_PATH/gaussdb -D $data_dir $run_mode & 2>&1 & sleep 10 } @@ -67,43 +68,45 @@ function start_primary_cluster() { primary_dn=$1 standby_dn=$2 ss_data=$3 + run_mode="-z cluster_primary" for node in $@ do if [ ${node} == ${ss_data} ]; then continue fi - gs_guc set -Z datanode -D $node -c "cluster_run_mode=cluster_primary" + run_mode="-z cluster_primary" done export DSS_HOME=${ss_data}/dss_home0 - start_dn $primary_dn + start_dn $primary_dn "$run_mode" export DSS_HOME=${ss_data}/dss_home1 - start_dn $standby_dn + start_dn $standby_dn "$run_mode" } function start_standby_cluster() { main_standby_dn=$1 standby_dn=$2 ss_data=$3 + run_mode="-z cluster_standby" for node in $@ do if [ ${node} == ${ss_data} ]; then continue fi - gs_guc set -Z datanode -D $node -c "cluster_run_mode=cluster_standby" + run_mode="-z cluster_standby" done - start_dn $main_standby_dn - start_dn $standby_dn + start_dn $main_standby_dn "$run_mode" + start_dn $standby_dn "$run_mode" } function assign_dorado_master_parameter() { for id in $@ do + gs_guc set -Z datanode -D ${SS_DATA_STANDBY}/dn${id} -c "xlog_file_path = '${SS_DATA}/dorado_shared_disk'" gs_guc set -Z datanode -D ${SS_DATA}/dn${id} -c "xlog_lock_file_path = '${SS_DATA}/shared_lock_primary'" gs_guc set -Z datanode -D ${SS_DATA}/dn${id} -c "application_name = 'master_${id}'" gs_guc set -Z datanode -D ${SS_DATA}/dn${id} -c "cross_cluster_replconninfo1='localhost=127.0.0.1 localport=${PGPORT[id]} remotehost=127.0.0.1 remoteport=${STANDBY_PGPORT[0]}'" gs_guc set -Z datanode -D ${SS_DATA}/dn${id} -c "cross_cluster_replconninfo2='localhost=127.0.0.1 localport=${PGPORT[id]} remotehost=127.0.0.1 remoteport=${STANDBY_PGPORT[1]}'" - gs_guc set -Z datanode -D ${SS_DATA}/dn${id} -c "cluster_run_mode = 'cluster_primary'" gs_guc set -Z datanode -D ${SS_DATA}/dn${id} -c "ha_module_debug = off" gs_guc set -Z datanode -D ${SS_DATA}/dn${id} -c "ss_log_level = 255" gs_guc set -Z datanode -D ${SS_DATA}/dn${id} -c "ss_log_backup_file_count = 100" @@ -120,7 +123,6 @@ function assign_dorado_standby_parameter() gs_guc set -Z datanode -D ${SS_DATA_STANDBY}/dn${id} -c "application_name = 'standby_${id}'" gs_guc set -Z datanode -D ${SS_DATA_STANDBY}/dn${id} -c "cross_cluster_replconninfo1='localhost=127.0.0.1 localport=${STANDBY_PGPORT[id]} remotehost=127.0.0.1 remoteport=${PGPORT[0]}'" gs_guc set -Z datanode -D ${SS_DATA_STANDBY}/dn${id} -c "cross_cluster_replconninfo2='localhost=127.0.0.1 localport=${STANDBY_PGPORT[id]} remotehost=127.0.0.1 remoteport=${PGPORT[1]}'" - gs_guc set -Z datanode -D ${SS_DATA_STANDBY}/dn${id} -c "cluster_run_mode = 'cluster_standby'" gs_guc set -Z datanode -D ${SS_DATA_STANDBY}/dn${id} -c "ha_module_debug = off" gs_guc set -Z datanode -D ${SS_DATA_STANDBY}/dn${id} -c "ss_log_level = 255" gs_guc set -Z datanode -D ${SS_DATA_STANDBY}/dn${id} -c "ss_log_backup_file_count = 100" From 9f20b710543caf1a26c5f89eedd38a1daf64ae5a Mon Sep 17 00:00:00 2001 From: zhang_xubo <2578876417@qq.com> Date: Mon, 14 Aug 2023 21:52:15 +0800 Subject: [PATCH 135/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=8D=87=E7=BA=A7?= =?UTF-8?q?=E4=B8=AD=E6=89=A7=E8=A1=8C=20select=20nextval=E5=AF=BC?= =?UTF-8?q?=E8=87=B4core=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../backend/utils/cache/knl_localpartdefcache.cpp | 3 +++ .../backend/utils/cache/knl_localsystupcache.cpp | 9 ++++++--- .../backend/utils/cache/knl_localtabdefcache.cpp | 3 +++ .../optimizer/commands/sequence/sequence.cpp | 10 +++++++++- 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/common/backend/utils/cache/knl_localpartdefcache.cpp b/src/common/backend/utils/cache/knl_localpartdefcache.cpp index bd94224f2..9f1046955 100644 --- a/src/common/backend/utils/cache/knl_localpartdefcache.cpp +++ b/src/common/backend/utils/cache/knl_localpartdefcache.cpp @@ -77,6 +77,9 @@ Partition LocalPartDefCache::SearchPartitionFromGlobalCopy(Oid part_oid) if (unlikely(!IsPrimaryRecoveryFinished())) { return NULL; } + if (unlikely(u_sess->attr.attr_common.IsInplaceUpgrade)) { + return NULL; + } uint32 hash_value = oid_hash((void *)&(part_oid), sizeof(Oid)); ResourceOwnerEnlargeGlobalBaseEntry(LOCAL_SYSDB_RESOWNER); GlobalPartitionEntry *global = (GlobalPartitionEntry *)m_global_partdefcache->SearchReadOnly(part_oid, hash_value); diff --git a/src/common/backend/utils/cache/knl_localsystupcache.cpp b/src/common/backend/utils/cache/knl_localsystupcache.cpp index 3755cbe8d..9309b5233 100644 --- a/src/common/backend/utils/cache/knl_localsystupcache.cpp +++ b/src/common/backend/utils/cache/knl_localsystupcache.cpp @@ -434,7 +434,8 @@ LocalCatCTup *LocalSysTupCache::SearchTupleFromGlobal(Datum *arguments, uint32 h bool bypass_gsc = HistoricSnapshotActive() || m_global_systupcache->enable_rls || !g_instance.global_sysdbcache.hot_standby || - unlikely(!IsPrimaryRecoveryFinished()); + unlikely(!IsPrimaryRecoveryFinished()) || + (unlikely(u_sess->attr.attr_common.IsInplaceUpgrade)); if (invalid_entries.ExistTuple(hash_value) || bypass_gsc) { global_ct = m_global_systupcache->SearchTupleFromFile(hash_value, arguments, true); } else { @@ -586,7 +587,8 @@ LocalCatCList *LocalSysTupCache::SearchListFromGlobal(int nkeys, Datum *argument bool bypass_gsc = HistoricSnapshotActive() || m_global_systupcache->enable_rls || !g_instance.global_sysdbcache.hot_standby || - unlikely(!IsPrimaryRecoveryFinished()); + unlikely(!IsPrimaryRecoveryFinished()) || + unlikely(u_sess->attr.attr_common.IsInplaceUpgrade); GlobalCatCList *global_cl; if (invalid_entries.ExistList() || bypass_gsc) { global_cl = m_global_systupcache->SearchListFromFile(hash_value, nkeys, arguments, true); @@ -704,7 +706,8 @@ LocalCatCTup *LocalSysTupCache::SearchTupleFromGlobalForProcAllArgs( bool bypass_gsc = HistoricSnapshotActive() || m_global_systupcache->enable_rls || !g_instance.global_sysdbcache.hot_standby || - unlikely(!IsPrimaryRecoveryFinished()); + unlikely(!IsPrimaryRecoveryFinished()) || + unlikely(u_sess->attr.attr_common.IsInplaceUpgrade); if (invalid_entries.ExistTuple(hash_value) || bypass_gsc) { global_ct = m_global_systupcache->SearchTupleFromFileWithArgModes(hash_value, arguments, argModes, true); } else { diff --git a/src/common/backend/utils/cache/knl_localtabdefcache.cpp b/src/common/backend/utils/cache/knl_localtabdefcache.cpp index 28c51f9fe..46a59cb33 100644 --- a/src/common/backend/utils/cache/knl_localtabdefcache.cpp +++ b/src/common/backend/utils/cache/knl_localtabdefcache.cpp @@ -119,6 +119,9 @@ Relation LocalTabDefCache::SearchRelationFromGlobalCopy(Oid rel_oid) if (unlikely(!IsPrimaryRecoveryFinished())) { return NULL; } + if (unlikely(u_sess->attr.attr_common.IsInplaceUpgrade)) { + return NULL; + } uint32 hash_value = oid_hash((void *)&(rel_oid), sizeof(Oid)); Index hash_index = HASH_INDEX(hash_value, (uint32)m_nbuckets); ResourceOwner owner = LOCAL_SYSDB_RESOWNER; diff --git a/src/gausskernel/optimizer/commands/sequence/sequence.cpp b/src/gausskernel/optimizer/commands/sequence/sequence.cpp index aa2a7ea67..3255a854c 100644 --- a/src/gausskernel/optimizer/commands/sequence/sequence.cpp +++ b/src/gausskernel/optimizer/commands/sequence/sequence.cpp @@ -1389,7 +1389,15 @@ bool shouldReturnNumeric() break; } - return get_nextval_rettype() == NUMERICOID; + HeapTuple ftup = SearchSysCache1(PROCOID, ObjectIdGetDatum(NEXTVALFUNCOID)); + if (!HeapTupleIsValid(ftup)) { + ereport(ERROR, (errmsg("cache lookup failed for function %u", NEXTVALFUNCOID))); + } + Form_pg_proc pform = (Form_pg_proc)GETSTRUCT(ftup); + bool ret = pform->prorettype == NUMERICOID; + ReleaseSysCache(ftup); + + return ret; } Datum nextval_oid(PG_FUNCTION_ARGS) From 0522059a80dd8a7c0244fc4580bc76a81bc1074e Mon Sep 17 00:00:00 2001 From: hwhbj Date: Wed, 16 Aug 2023 10:45:58 +0800 Subject: [PATCH 136/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D22.03=E7=BC=96?= =?UTF-8?q?=E8=AF=91=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- configure | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure b/configure index 629c105b7..da13baaca 100755 --- a/configure +++ b/configure @@ -3279,7 +3279,7 @@ _ACEOF fi -if [ "$(cat /etc/system-release)" =~ ^"openEuler release 22.03".* ]; then +if [[ "$(cat /etc/system-release)" =~ ^"openEuler release 22.03".* ]]; then with_openeuler_major=yes fi From bdaa869ecc098be75e9341c89a699d578c770f54 Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Wed, 16 Aug 2023 14:12:20 +0800 Subject: [PATCH 137/304] =?UTF-8?q?=E6=97=A5=E5=BF=97=E5=90=88=E4=B8=80?= =?UTF-8?q?=E8=A1=A5=E5=85=852?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ddes/adapter/ss_reform_common.cpp | 37 +- .../storage/access/transam/xlog.cpp | 447 +++++++++--------- src/include/ddes/dms/ss_reform_common.h | 2 +- 3 files changed, 230 insertions(+), 256 deletions(-) diff --git a/src/gausskernel/ddes/adapter/ss_reform_common.cpp b/src/gausskernel/ddes/adapter/ss_reform_common.cpp index cca457cb5..25ab24c4e 100644 --- a/src/gausskernel/ddes/adapter/ss_reform_common.cpp +++ b/src/gausskernel/ddes/adapter/ss_reform_common.cpp @@ -64,7 +64,7 @@ std::vector SSGetAllStableNodeId() return posList; } -int SSXLogFileReadAnyTLI(XLogSegNo segno, int emode, uint32 sources, char* xlog_path) +int SSXLogFileOpenAnyTLI(XLogSegNo segno, int emode, uint32 sources, char* xlog_path) { char path[MAXPGPATH]; ListCell *cell = NULL; @@ -83,7 +83,7 @@ int SSXLogFileReadAnyTLI(XLogSegNo segno, int emode, uint32 sources, char* xlog_ t_thrd.xlog_cxt.restoredFromArchive = false; fd = BasicOpenFile(path, O_RDONLY | PG_BINARY, 0); -retry: + if (fd >= 0) { /* Success! */ t_thrd.xlog_cxt.curFileTLI = tli; @@ -103,38 +103,9 @@ int SSXLogFileReadAnyTLI(XLogSegNo segno, int emode, uint32 sources, char* xlog_ /* * When SS_CLUSTER_DORADO_REPLICATION enabled, current xlog dictionary may be not the correct dictionary, * because all xlog dictionaries are in the same LUN, we need loop over other dictionaries. - * Do we need source == XLOG_FROM_STREAM? */ - if (SS_CLUSTER_DORADO_REPLICATION) { - std::vector nodeList = SSGetAllStableNodeId(); // stable node list, - Assert(!nodeList.empty()); - char xlogPath[MAXPGPATH]; - char *dssdir = g_instance.attr.attr_storage.dss_attr.ss_dss_vg_name; - for (auto elem : nodeList) { - if (elem == g_instance.dms_cxt.SSReformerControl.recoveryInstId) { - continue; - } - - errorno = memset_s(xlogPath, sizeof(xlogPath), 0, sizeof(xlogPath)); - securec_check_ss(errorno, "", ""); - /* try to read from other xlog dictionary */ - errorno = snprintf_s(xlogPath, MAXPGPATH, MAXPGPATH - 1, "%s/pg_xlog%d", dssdir, elem); - securec_check_ss(errorno, "", ""); - - errorno = memset_s(path, sizeof(path), 0, sizeof(path)); - securec_check_ss(errorno, "", ""); - - errorno = snprintf_s(path, MAXPGPATH, MAXPGPATH - 1, "%s/%08X%08X%08X", xlogPath, tli, - (uint32)((segno) / XLogSegmentsPerXLogId), (uint32)((segno) % XLogSegmentsPerXLogId)); - securec_check_ss(errorno, "", ""); - - fd = BasicOpenFile(path, O_RDONLY | PG_BINARY, 0); - if (fd < 0) { - continue; - } - ereport(LOG, (errmsg("find xlog file in path : \"%s\"", path))); - goto retry; - } + if (fd < 0 && SS_CLUSTER_DORADO_REPLICATION) { + return -1; } if (!FILE_POSSIBLY_DELETED(errno)) { diff --git a/src/gausskernel/storage/access/transam/xlog.cpp b/src/gausskernel/storage/access/transam/xlog.cpp index 4bd91aed6..d297a1d01 100755 --- a/src/gausskernel/storage/access/transam/xlog.cpp +++ b/src/gausskernel/storage/access/transam/xlog.cpp @@ -419,6 +419,8 @@ static uint64 XLogRecPtrToBytePos(XLogRecPtr ptr); static XLogRecPtr XLogInsertRecordSingle(XLogRecData *rdata, XLogRecPtr fpw_lsn); static int SSReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *readTLI, char* xlog_path); +static int SSDoradoReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, + XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *readTLI, char* xlog_path); void ArchiveXlogForForceFinishRedo(XLogReaderState *xlogreader, TermFileData *term_file); TermFileData GetTermFileDataAndClear(void); XLogRecPtr mpfl_read_max_flush_lsn(); @@ -4202,7 +4204,7 @@ static int XLogFileOpenInternal(XLogSegNo segno, const char *xlog_dir) if (SS_CLUSTER_DORADO_REPLICATION) { fd = SSErgodicOpenXlogFile(segno, O_RDWR | PG_BINARY | (unsigned int)get_sync_bit(u_sess->attr.attr_storage.sync_method), - S_IRUSR | S_IWUSR); + S_IRUSR | S_IWUSR); } else { fd = BasicOpenFile(path, O_RDWR | PG_BINARY | (unsigned int)get_sync_bit(u_sess->attr.attr_storage.sync_method), S_IRUSR | S_IWUSR); @@ -5246,8 +5248,34 @@ static XLogRecord *ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, in for (;;) { char *errormsg = NULL; + if (SS_CLUSTER_DORADO_REPLICATION) { + char xlog_path_ergodic[MAXPGPATH]; + struct dirent *entry; + errno_t errorno = EOK; + DIR* dssdir = opendir(g_instance.attr.attr_storage.dss_attr.ss_dss_vg_name); + if (dssdir == NULL) { + ereport(PANIC, (errcode_for_file_access(), errmsg("Error opening dssdir %s", + g_instance.attr.attr_storage.dss_attr.ss_dss_vg_name))); + } + + while ((entry = readdir(dssdir)) != NULL) { + if (strncmp(entry->d_name, "pg_xlog", strlen("pg_xlog")) == 0) { + errorno = snprintf_s(xlog_path_ergodic, MAXPGPATH, MAXPGPATH - 1, "%s/%s", + g_instance.attr.attr_storage.dss_attr.ss_dss_vg_name, entry->d_name); + securec_check_ss(errorno, "", ""); + record = XLogReadRecord(xlogreader, RecPtr, &errormsg, true, xlog_path_ergodic); + if (record != NULL) { + break; + } + } else { + continue; + } + } + closedir(dssdir); + } else { + record = XLogReadRecord(xlogreader, RecPtr, &errormsg); + } - record = XLogReadRecord(xlogreader, RecPtr, &errormsg); t_thrd.xlog_cxt.ReadRecPtr = xlogreader->ReadRecPtr; t_thrd.xlog_cxt.EndRecPtr = xlogreader->EndRecPtr; g_instance.comm_cxt.predo_cxt.redoPf.read_ptr = t_thrd.xlog_cxt.ReadRecPtr; @@ -9585,7 +9613,7 @@ void StartupXLOG(void) } } - if (SS_PRIMARY_MODE || SS_STANDBY_CLUSTER_MAIN_STANDBY) { + if (SS_PRIMARY_MODE || IS_SS_REPLICATION_MAIN_STANBY_NODE) { g_instance.dms_cxt.SSReformerControl.clusterRunMode = (ClusterRunMode)g_instance.attr.attr_common.cluster_run_mode; SSSaveReformerCtrl(); } @@ -19194,19 +19222,125 @@ static void SSOndemandXlogCopy(XLogSegNo copySegNo, uint32 startOffset, char *co t_thrd.ondemand_xlog_copy_cxt.openLogOff += copyBytes; } -static int SSReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, - XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *readTLI, char* xlog_path) +static int SSReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int expectReadLen, + XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *readTLI, char* xlog_path) { /* Load reader private data */ XLogPageReadPrivate *readprivate = (XLogPageReadPrivate *)xlogreader->private_data; int emode = IsExtremeRedo() ? LOG : readprivate->emode; + bool randAccess = IsExtremeRedo() ? false : readprivate->randAccess; + XLogRecPtr RecPtr = targetPagePtr; + uint32 targetPageOff; + uint32 actualBytes; + +#ifdef USE_ASSERT_CHECKING + XLogSegNo targetSegNo; + + XLByteToSeg(targetPagePtr, targetSegNo); +#endif + targetPageOff = targetPagePtr % XLogSegSize; + + /* + * See if we need to switch to a new segment because the requested record + * is not in the currently open one. + */ + if (t_thrd.xlog_cxt.readFile >= 0 && !XLByteInSeg(targetPagePtr, t_thrd.xlog_cxt.readSegNo)) { + close(t_thrd.xlog_cxt.readFile); + t_thrd.xlog_cxt.readFile = -1; + t_thrd.xlog_cxt.readSource = 0; + } + + XLByteToSeg(targetPagePtr, t_thrd.xlog_cxt.readSegNo); + XLByteAdvance(RecPtr, expectReadLen); + + /* In archive or crash recovery. */ + if (t_thrd.xlog_cxt.readFile < 0) { + uint32 sources; + + /* Reset curFileTLI if random fetch. */ + if (randAccess) { + t_thrd.xlog_cxt.curFileTLI = 0; + } + + sources = XLOG_FROM_PG_XLOG; + if (t_thrd.xlog_cxt.InArchiveRecovery) { + sources |= XLOG_FROM_ARCHIVE; + } + + t_thrd.xlog_cxt.readFile = SSXLogFileOpenAnyTLI(t_thrd.xlog_cxt.readSegNo, emode, sources, xlog_path); + + if (t_thrd.xlog_cxt.readFile < 0) { + return -1; + } + } + + /* + * At this point, we have the right segment open and if we're streaming we + * know the requested record is in it. + */ + Assert(t_thrd.xlog_cxt.readFile != -1); + + /* read size for XLOG_FROM_PG_XLOG */ + t_thrd.xlog_cxt.readLen = XLOG_BLCKSZ; + + /* Read the requested page */ + t_thrd.xlog_cxt.readOff = targetPageOff; + + + if (xlogreader->preReadBuf == NULL) { + actualBytes = (uint32)pread(t_thrd.xlog_cxt.readFile, readBuf, t_thrd.xlog_cxt.readLen, t_thrd.xlog_cxt.readOff); + } else { + actualBytes = (uint32)SSReadXlogInternal(xlogreader, targetPagePtr, targetRecPtr, readBuf, t_thrd.xlog_cxt.readLen); + } + + if (actualBytes != t_thrd.xlog_cxt.readLen) { + ereport(LOG, (errcode_for_file_access(), errmsg("read xlog(start:%X/%X, pos:%u len:%d) failed : %m", + static_cast(targetPagePtr >> BIT_NUM_INT32), + static_cast(targetPagePtr), targetPageOff, + expectReadLen))); + ereport(emode_for_corrupt_record(emode, RecPtr), + (errcode_for_file_access(), + errmsg("could not read from log file %s to offset %u: %m", + XLogFileNameP(t_thrd.xlog_cxt.ThisTimeLineID, t_thrd.xlog_cxt.readSegNo), + t_thrd.xlog_cxt.readOff))); + goto next_record_is_invalid; + } + + Assert(targetSegNo == t_thrd.xlog_cxt.readSegNo); + Assert(targetPageOff == t_thrd.xlog_cxt.readOff); + Assert((uint32)expectReadLen <= t_thrd.xlog_cxt.readLen); + + *readTLI = t_thrd.xlog_cxt.curFileTLI; + + return (int)t_thrd.xlog_cxt.readLen; + +next_record_is_invalid: + t_thrd.xlog_cxt.failedSources |= t_thrd.xlog_cxt.readSource; + + if (t_thrd.xlog_cxt.readFile >= 0) { + close(t_thrd.xlog_cxt.readFile); + } + t_thrd.xlog_cxt.readFile = -1; + t_thrd.xlog_cxt.readLen = 0; + t_thrd.xlog_cxt.readSource = 0; + + return -1; +} + +static int SSDoradoReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, + XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *readTLI, char* xlog_path) +{ + /* Load reader private data */ + XLogPageReadPrivate *readprivate = (XLogPageReadPrivate*)xlogreader->private_data; + int emode = readprivate->emode; XLogRecPtr RecPtr = targetPagePtr; uint32 targetPageOff; bool processtrxn = false; bool fetching_ckpt = readprivate->fetching_ckpt; - bool randAccess = IsExtremeRedo() ? false : readprivate->randAccess; + bool randAccess = readprivate->randAccess; XLogCtlData *xlogctl = t_thrd.shemem_ptr_cxt.XLogCtl; XLogSegNo replayedSegNo; + uint32 sources; uint32 actualBytes; #ifdef USE_ASSERT_CHECKING @@ -19241,22 +19375,25 @@ static int SSReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int t_thrd.xlog_cxt.readSource = 0; } + t_thrd.xlog_cxt.readOff = targetPageOff; + t_thrd.xlog_cxt.readLen = XLOG_BLCKSZ; + t_thrd.xlog_cxt.receivedUpto = g_instance.xlog_cxt.ssReplicationXLogCtl->insertHead; XLByteToSeg(targetPagePtr, t_thrd.xlog_cxt.readSegNo); XLByteAdvance(RecPtr, reqLen); - + + XLogRecPtr expectedRecPtr = RecPtr; + if (RecPtr % XLogSegSize == 0) { + XLByteAdvance(expectedRecPtr, SizeOfXLogLongPHD); + } else if (RecPtr % XLOG_BLCKSZ == 0) { + XLByteAdvance(expectedRecPtr, SizeOfXLogShortPHD); + } + retry: /* See if we need to retrieve more data */ /* In ss dorado replication, we don't start walrecwrite thread, so t_thrd.xlog_cxt.receivedUpto = 0 */ if (t_thrd.xlog_cxt.readFile < 0 || (t_thrd.xlog_cxt.readSource == XLOG_FROM_STREAM && XLByteLT(t_thrd.xlog_cxt.receivedUpto, RecPtr))) { - if (t_thrd.xlog_cxt.StandbyMode && t_thrd.xlog_cxt.startup_processing && (DORADO_STANDBY_CLUSTER || SS_CLUSTER_DORADO_REPLICATION)) { - - /* - * In standby mode, wait for the requested record to become - * available, either via restore_command succeeding to restore the - * segment, or via walreceiver having streamed the record. - */ - + if (t_thrd.xlog_cxt.StandbyMode && t_thrd.xlog_cxt.startup_processing) { for (;;) { /* * Need to check here also for the case where consistency level is @@ -19271,9 +19408,7 @@ static int SSReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int CheckRecoveryConsistency(); if (WalRcvInProgress()) { - XLogRecPtr expectedRecPtr = RecPtr; bool havedata = false; - /* * If we find an invalid record in the WAL streamed from * master, something is seriously wrong. There's little @@ -19308,61 +19443,38 @@ static int SSReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int * XLogReceiptTime will not advance, so the grace time * alloted to conflicting queries will decrease. */ - if (RecPtr % XLogSegSize == 0) { - XLByteAdvance(expectedRecPtr, SizeOfXLogLongPHD); - } else if (RecPtr % XLOG_BLCKSZ == 0) { - XLByteAdvance(expectedRecPtr, SizeOfXLogShortPHD); - } + if (XLByteLT(expectedRecPtr, t_thrd.xlog_cxt.receivedUpto)) { havedata = true; } else { havedata = false; XLogRecPtr latestChunkStart; - if (SS_CLUSTER_DORADO_REPLICATION) { - ReadSSDoradoCtlInfoFile(); - t_thrd.xlog_cxt.receivedUpto = g_instance.xlog_cxt.ssReplicationXLogCtl->insertHead; - latestChunkStart = g_instance.xlog_cxt.ssReplicationXLogCtl->insertHead; - } else { - t_thrd.xlog_cxt.receivedUpto = GetWalRcvWriteRecPtr(&latestChunkStart); - } + ReadSSDoradoCtlInfoFile(); + t_thrd.xlog_cxt.receivedUpto = g_instance.xlog_cxt.ssReplicationXLogCtl->insertHead; + latestChunkStart = g_instance.xlog_cxt.ssReplicationXLogCtl->insertHead; if (XLByteLT(expectedRecPtr, t_thrd.xlog_cxt.receivedUpto)) { - havedata = true; - if (!XLByteLT(RecPtr, latestChunkStart)) { - t_thrd.xlog_cxt.XLogReceiptTime = GetCurrentTimestamp(); - SetCurrentChunkStartTime(t_thrd.xlog_cxt.XLogReceiptTime); - } - } else { - havedata = false; + havedata = true; + if (!XLByteLT(RecPtr, latestChunkStart)) { + t_thrd.xlog_cxt.XLogReceiptTime = GetCurrentTimestamp(); + SetCurrentChunkStartTime(t_thrd.xlog_cxt.XLogReceiptTime); } + } else { + havedata = false; + } } if (havedata) { - /* - * Great, streamed far enough. Open the file if it's - * not open already. Use XLOG_FROM_STREAM so that - * source info is set correctly and XLogReceiptTime - * isn't changed. - */ - if (t_thrd.xlog_cxt.readFile < 0) { - t_thrd.xlog_cxt.readFile = XLogFileRead(t_thrd.xlog_cxt.readSegNo, PANIC, - t_thrd.xlog_cxt.recoveryTargetTLI, XLOG_FROM_STREAM, - false); - Assert(t_thrd.xlog_cxt.readFile >= 0); - } else { - /* just make sure source info is correct... */ - t_thrd.xlog_cxt.readSource = XLOG_FROM_STREAM; - t_thrd.xlog_cxt.XLogReceiptSource = XLOG_FROM_STREAM; - } - + t_thrd.xlog_cxt.readSource = XLOG_FROM_STREAM; + t_thrd.xlog_cxt.XLogReceiptSource = XLOG_FROM_STREAM; break; } t_thrd.xlog_cxt.RedoDone = IsRedoDonePromoting(); pg_memory_barrier(); - if (DORADO_STANDBY_CLUSTER_MAINSTANDBY_NODE && WalRcvIsDone() && CheckForFailoverTrigger()) { + if (IS_SS_REPLICATION_MAIN_STANBY_NODE && WalRcvIsDone() && CheckForFailoverTrigger()) { t_thrd.xlog_cxt.receivedUpto = GetWalRcvWriteRecPtr(NULL); if (XLByteLT(RecPtr, t_thrd.xlog_cxt.receivedUpto)) { /* wait xlog redo done */ @@ -19370,31 +19482,22 @@ static int SSReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int } ProcTxnWorkLoad(true); - /* use volatile pointer to prevent code rearrangement */ - volatile WalRcvData *walrcv = t_thrd.walreceiverfuncs_cxt.WalRcv; - SpinLockAcquire(&walrcv->mutex); - walrcv->dummyStandbyConnectFailed = false; - SpinLockRelease(&walrcv->mutex); - - ereport(LOG, (errmsg("RecPtr(%X/%X),receivedUpto(%X/%X)", (uint32)(RecPtr >> 32), + ereport(LOG, (errmsg("RecPtr(%X/%X), receivedUpto(%X/%X).", (uint32)(RecPtr >> 32), (uint32)RecPtr, (uint32)(t_thrd.xlog_cxt.receivedUpto >> 32), (uint32)t_thrd.xlog_cxt.receivedUpto))); ShutdownWalRcv(); ShutdownDataRcv(); - goto triggered; } - if (IS_SHARED_STORAGE_MODE || SS_CLUSTER_DORADO_REPLICATION) { - uint32 disableConnectionNode = - pg_atomic_read_u32(&g_instance.comm_cxt.localinfo_cxt.need_disable_connection_node); - if (disableConnectionNode && WalRcvIsRunning()) { - ereport(LOG, (errmsg("request xlog receivedupto at %X/%X.", - (uint32)(t_thrd.xlog_cxt.receivedUpto >> 32), - (uint32)t_thrd.xlog_cxt.receivedUpto))); - ShutdownWalRcv(); - } + uint32 disableConnectionNode = + pg_atomic_read_u32(&g_instance.comm_cxt.localinfo_cxt.need_disable_connection_node); + if (disableConnectionNode && WalRcvIsRunning()) { + ereport(LOG, (errmsg("request xlog receivedupto at %X/%X.", + (uint32)(t_thrd.xlog_cxt.receivedUpto >> 32), + (uint32)t_thrd.xlog_cxt.receivedUpto))); + ShutdownWalRcv(); } if (!processtrxn) { ProcTxnWorkLoad(true); @@ -19408,11 +19511,6 @@ static int SSReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int processtrxn = false; ResetLatch(&t_thrd.shemem_ptr_cxt.XLogCtl->recoveryWakeupLatch); } else { - uint32 sources; - /* - * Until walreceiver manages to reconnect, poll the - * archive. - */ if (t_thrd.xlog_cxt.readFile >= 0) { close(t_thrd.xlog_cxt.readFile); t_thrd.xlog_cxt.readFile = -1; @@ -19427,12 +19525,7 @@ static int SSReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int * existing file from pg_xlog. */ sources = XLOG_FROM_ARCHIVE | XLOG_FROM_PG_XLOG; - ereport(DEBUG5, (errmsg("failedSources: %u", t_thrd.xlog_cxt.failedSources))); - if (!(sources & ~t_thrd.xlog_cxt.failedSources)) { - /* - * We've exhausted all options for retrieving the - * file. Retry. - */ + if (XLByteLT(t_thrd.xlog_cxt.receivedUpto, expectedRecPtr)) { t_thrd.xlog_cxt.failedSources = 0; /* @@ -19446,21 +19539,21 @@ static int SSReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int } } - if (t_thrd.startup_cxt.shutdown_requested) { - ereport(LOG, (errmsg("startup shutdown"))); - proc_exit(0); - } - if (!xlogctl->IsRecoveryDone) { g_instance.comm_cxt.predo_cxt.redoPf.redo_done_time = GetCurrentTimestamp(); g_instance.comm_cxt.predo_cxt.redoPf.recovery_done_ptr = t_thrd.xlog_cxt.ReadRecPtr; ereport(LOG, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), - errmsg("XLogPageRead IsRecoveryDone is set true," - "ReadRecPtr:%X/%X, EndRecPtr:%X/%X", - (uint32)(t_thrd.xlog_cxt.ReadRecPtr >> 32), - (uint32)(t_thrd.xlog_cxt.ReadRecPtr), - (uint32)(t_thrd.xlog_cxt.EndRecPtr >> 32), - (uint32)(t_thrd.xlog_cxt.EndRecPtr)))); + errmsg("XLogPageRead IsRecoveryDone is set true, " + "ReadRecPtr:%X/%X, EndRecPtr:%X/%X, " + "receivedUpto:%X/%X, CtlInfo_insertHead:%X/%X.", + (uint32)(t_thrd.xlog_cxt.ReadRecPtr >> 32), + (uint32)(t_thrd.xlog_cxt.ReadRecPtr), + (uint32)(t_thrd.xlog_cxt.EndRecPtr >> 32), + (uint32)(t_thrd.xlog_cxt.EndRecPtr), + (uint32)(t_thrd.xlog_cxt.receivedUpto >> 32), + (uint32)(t_thrd.xlog_cxt.receivedUpto), + (uint32)(g_instance.xlog_cxt.ssReplicationXLogCtl->insertHead >> 32), + (uint32)(g_instance.xlog_cxt.ssReplicationXLogCtl->insertHead)))); parallel_recovery::redo_dump_all_stats(); } @@ -19471,7 +19564,7 @@ static int SSReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int ProcTxnWorkLoad(true); if (!xlogctl->IsRecoveryDone) { SendPostmasterSignal(PMSIGNAL_LOCAL_RECOVERY_DONE); - if ((DORADO_STANDBY_CLUSTER || SS_REPLICATION_STANDBY_CLUSTER) && SS_PERFORMING_SWITCHOVER) { + if (SS_PERFORMING_SWITCHOVER) { g_instance.dms_cxt.SSClusterState = NODESTATE_STANDBY_PROMOTED; } } @@ -19513,47 +19606,30 @@ static int SSReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int */ load_server_mode(); - if (DORADO_STANDBY_CLUSTER_MAINSTANDBY_NODE && CheckForFailoverTrigger()) { + if (IS_SS_REPLICATION_MAIN_STANBY_NODE && CheckForFailoverTrigger()) { goto triggered; - } else if (DORADO_STANDBY_CLUSTER_MAINSTANDBY_NODE || IS_SS_REPLICATION_MAIN_STANBY_NODE) { - ProcTxnWorkLoad(false); - /* use volatile pointer to prevent code rearrangement */ - volatile WalRcvData *walrcv = t_thrd.walreceiverfuncs_cxt.WalRcv; -#ifndef ENABLE_MULTIPLE_NODES - rename_recovery_conf_for_roach(); -#endif + } + ProcTxnWorkLoad(false); + /* use volatile pointer to prevent code rearrangement */ + volatile WalRcvData *walrcv = t_thrd.walreceiverfuncs_cxt.WalRcv; + rename_recovery_conf_for_roach(); - ereport(LOG, (errmsg("request xlog stream from shared storage at %X/%X.", - fetching_ckpt ? (uint32)(t_thrd.xlog_cxt.RedoStartLSN >> 32) - : (uint32)(targetRecPtr >> 32), - fetching_ckpt ? (uint32)t_thrd.xlog_cxt.RedoStartLSN - : (uint32)targetRecPtr))); - ShutdownWalRcv(); - t_thrd.xlog_cxt.receivedUpto = 0; - SpinLockAcquire(&walrcv->mutex); - walrcv->receivedUpto = 0; - SpinLockRelease(&walrcv->mutex); + ereport(LOG, (errmsg("request xlog stream from shared storage at %X/%X.", + fetching_ckpt ? (uint32)(t_thrd.xlog_cxt.RedoStartLSN >> 32) + : (uint32)(targetRecPtr >> 32), + fetching_ckpt ? (uint32)t_thrd.xlog_cxt.RedoStartLSN + : (uint32)targetRecPtr))); + ShutdownWalRcv(); + SpinLockAcquire(&walrcv->mutex); + walrcv->receivedUpto = 0; + SpinLockRelease(&walrcv->mutex); - RequestXLogStreaming(fetching_ckpt ? &t_thrd.xlog_cxt.RedoStartLSN : &targetRecPtr, 0, - REPCONNTARGET_SHARED_STORAGE, 0); - continue; - } - } - /* Don't try to read from a source that just failed */ - sources &= ~t_thrd.xlog_cxt.failedSources; - t_thrd.xlog_cxt.readFile = SSXLogFileReadAnyTLI(t_thrd.xlog_cxt.readSegNo, emode, sources, xlog_path); - if (t_thrd.xlog_cxt.readFile >= 0) { - break; + RequestXLogStreaming(fetching_ckpt ? &t_thrd.xlog_cxt.RedoStartLSN : &targetRecPtr, 0, + REPCONNTARGET_SHARED_STORAGE, 0); + continue; } - ereport(DEBUG5, (errmsg("do not find any more files.sources=%u failedSources=%u", sources, - t_thrd.xlog_cxt.failedSources))); - /* - * Nope, not found in archive and/or pg_xlog. - */ - t_thrd.xlog_cxt.failedSources |= sources; - - if (DORADO_STANDBY_CLUSTER_MAINSTANDBY_NODE && CheckForFailoverTrigger()) { + if (CheckForFailoverTrigger()) { XLogRecPtr receivedUpto = GetWalRcvWriteRecPtr(NULL); XLogRecPtr EndRecPtrTemp = t_thrd.xlog_cxt.EndRecPtr; XLByteAdvance(EndRecPtrTemp, SizeOfXLogRecord); @@ -19578,7 +19654,7 @@ static int SSReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int t_thrd.xlog_cxt.readOff, t_thrd.xlog_cxt.readLen))); goto triggered; } - + break; } /* @@ -19587,100 +19663,21 @@ static int SSReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int */ RedoInterruptCallBack(); } - } else { - /* In archive or crash recovery. */ - if (t_thrd.xlog_cxt.readFile < 0) { - uint32 sources; - - /* Reset curFileTLI if random fetch. */ - if (randAccess) { - t_thrd.xlog_cxt.curFileTLI = 0; - } - - sources = XLOG_FROM_PG_XLOG; - if (t_thrd.xlog_cxt.InArchiveRecovery) { - sources |= XLOG_FROM_ARCHIVE; - } - - t_thrd.xlog_cxt.readFile = SSXLogFileReadAnyTLI(t_thrd.xlog_cxt.readSegNo, - emode, sources, xlog_path); - - if (t_thrd.xlog_cxt.readFile < 0) { - return -1; - } - } } } - /* - * At this point, we have the right segment open and if we're streaming we - * know the requested record is in it. - */ - Assert(t_thrd.xlog_cxt.readFile != -1); - - if (t_thrd.xlog_cxt.readSource == XLOG_FROM_STREAM) { - if ((targetPagePtr / XLOG_BLCKSZ) != (t_thrd.xlog_cxt.receivedUpto / XLOG_BLCKSZ)) { - t_thrd.xlog_cxt.readLen = XLOG_BLCKSZ; - } else { - t_thrd.xlog_cxt.readLen = t_thrd.xlog_cxt.receivedUpto % XLogSegSize - targetPageOff; + if (t_thrd.xlog_cxt.readFile < 0) { + t_thrd.xlog_cxt.readFile = SSXLogFileOpenAnyTLI(t_thrd.xlog_cxt.readSegNo, emode, sources, xlog_path); + if (t_thrd.xlog_cxt.readFile < 0) { + ereport(LOG, (errmsg("%s read failed, change xlog file.", xlog_path))); + goto next_record_is_invalid; } - } else { - t_thrd.xlog_cxt.readLen = XLOG_BLCKSZ; } - /* Read the requested page */ - t_thrd.xlog_cxt.readOff = targetPageOff; - - if (DORADO_STANDBY_CLUSTER_MAINSTANDBY_NODE || IS_SS_REPLICATION_MAIN_STANBY_NODE) { -try_again: - if (lseek(t_thrd.xlog_cxt.readFile, (off_t)t_thrd.xlog_cxt.readOff, SEEK_SET) < 0) { - ereport(emode_for_corrupt_record(emode, RecPtr), - (errcode_for_file_access(), - errmsg("could not seek in log file %s to offset %u: %m", - XLogFileNameP(t_thrd.xlog_cxt.ThisTimeLineID, t_thrd.xlog_cxt.readSegNo), - t_thrd.xlog_cxt.readOff))); - if (errno == EINTR) { - errno = 0; - pg_usleep(1000); - goto try_again; - } - goto next_record_is_invalid; - } - pgstat_report_waitevent(WAIT_EVENT_WAL_READ); - actualBytes = read(t_thrd.xlog_cxt.readFile, readBuf, XLOG_BLCKSZ); - pgstat_report_waitevent(WAIT_EVENT_END); - if (actualBytes != XLOG_BLCKSZ) { - ereport(emode_for_corrupt_record(emode, RecPtr), - (errcode_for_file_access(), - errmsg("could not read from log file %s to offset %u: %m", - XLogFileNameP(t_thrd.xlog_cxt.ThisTimeLineID, t_thrd.xlog_cxt.readSegNo), - t_thrd.xlog_cxt.readOff))); - if (errno == EINTR) { - errno = 0; - pg_usleep(1000); - goto try_again; - } - goto next_record_is_invalid; - } - } else { - if (xlogreader->preReadBuf != NULL) { - actualBytes = (uint32)SSReadXlogInternal(xlogreader, targetPagePtr, targetRecPtr, readBuf, XLOG_BLCKSZ); - } else { - actualBytes = (uint32)pread(t_thrd.xlog_cxt.readFile, readBuf, XLOG_BLCKSZ, t_thrd.xlog_cxt.readOff); - } - - if (actualBytes != XLOG_BLCKSZ) { - ereport(LOG, (errcode_for_file_access(), errmsg("read xlog(start:%X/%X, pos:%u len:%d) failed : %m", - static_cast(targetPagePtr >> BIT_NUM_INT32), - static_cast(targetPagePtr), targetPageOff, - reqLen))); - ereport(emode_for_corrupt_record(emode, RecPtr), - (errcode_for_file_access(), - errmsg("could not read from log file %s to offset %u: %m", - XLogFileNameP(t_thrd.xlog_cxt.ThisTimeLineID, t_thrd.xlog_cxt.readSegNo), - t_thrd.xlog_cxt.readOff))); - goto next_record_is_invalid; - } + actualBytes = (uint32)pread(t_thrd.xlog_cxt.readFile, readBuf, t_thrd.xlog_cxt.readLen, t_thrd.xlog_cxt.readOff); + if (actualBytes != t_thrd.xlog_cxt.readLen) { + ereport(LOG, (errmsg("%s read failed, change xlog file.", xlog_path))); + goto next_record_is_invalid; } Assert(targetSegNo == t_thrd.xlog_cxt.readSegNo); @@ -19715,8 +19712,14 @@ static int SSReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int int SSXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *readTLI, char* xlog_path) -{ - int read_len = SSReadXLog(xlogreader, targetPagePtr, reqLen, targetRecPtr, - readBuf, readTLI, g_instance.dms_cxt.SSRecoveryInfo.recovery_xlog_dir); +{ + int read_len; + if (SS_CLUSTER_DORADO_REPLICATION) { + read_len = SSDoradoReadXLog(xlogreader, targetPagePtr, reqLen, targetRecPtr, + readBuf, readTLI, xlog_path); + } else { + read_len = SSReadXLog(xlogreader, targetPagePtr, reqLen, targetRecPtr, + readBuf, readTLI, g_instance.dms_cxt.SSRecoveryInfo.recovery_xlog_dir); + } return read_len; } diff --git a/src/include/ddes/dms/ss_reform_common.h b/src/include/ddes/dms/ss_reform_common.h index a32f69dc7..3d7118340 100644 --- a/src/include/ddes/dms/ss_reform_common.h +++ b/src/include/ddes/dms/ss_reform_common.h @@ -46,5 +46,5 @@ void SSReadControlFile(int id, bool updateDmsCtx = false); void SSClearSegCache(); int SSCancelTransactionOfAllStandby(SSBroadcastOp type); int SSProcessCancelTransaction(SSBroadcastOp type); -int SSXLogFileReadAnyTLI(XLogSegNo segno, int emode, uint32 sources, char* xlog_path); +int SSXLogFileOpenAnyTLI(XLogSegNo segno, int emode, uint32 sources, char* xlog_path); void SSStandbySetLibpqswConninfo(); From d346dd2b40dbb3967745ef15587f590f4262343e Mon Sep 17 00:00:00 2001 From: chenchao <1790599142@qq.com> Date: Wed, 16 Aug 2023 16:16:30 +0800 Subject: [PATCH 138/304] =?UTF-8?q?=E6=9B=B4=E6=96=B0dss=20conmmit?= =?UTF-8?q?=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/ddes/ddes_commit_id | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gausskernel/ddes/ddes_commit_id b/src/gausskernel/ddes/ddes_commit_id index d68e79777..4677baf34 100644 --- a/src/gausskernel/ddes/ddes_commit_id +++ b/src/gausskernel/ddes/ddes_commit_id @@ -1,2 +1,2 @@ dms_commit_id=b12ceed7bb0d3de049d8b4858180c77110ff6444 -dss_commit_id=f9bb2fa60f02331b0373559776a8ebdba86c002f +dss_commit_id=0b9c9bc1a2c3cac463ea70fd6abc1790e9f125aa From 72c6f6ea2f5b2f7f158f70e34f1a9f6311cbc404 Mon Sep 17 00:00:00 2001 From: "arcoalien@qq.com" Date: Wed, 16 Aug 2023 11:18:36 +0800 Subject: [PATCH 139/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dpg=5Flogical=5Fslot?= =?UTF-8?q?=5Fget=5Fchanges=E6=8A=A5=E9=94=99=E7=9A=84=E9=97=AE=E9=A2=98?= =?UTF-8?q?=EF=BC=8C=E8=A1=A5=E5=85=85=E9=94=81=E8=B6=85=E6=97=B6=E7=9A=84?= =?UTF-8?q?=E5=A0=86=E6=A0=88=E6=89=93=E5=8D=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/utils/resowner/resowner.cpp | 11 ++++++++++- .../cbb/instruments/gs_stack/gs_stack.cpp | 15 +++++++++++++-- src/gausskernel/storage/lmgr/proc.cpp | 4 ++++ .../storage/replication/logical/logicalfuncs.cpp | 2 +- src/include/utils/resowner.h | 1 + 5 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/common/backend/utils/resowner/resowner.cpp b/src/common/backend/utils/resowner/resowner.cpp index 41eee7d72..9eb9b8787 100755 --- a/src/common/backend/utils/resowner/resowner.cpp +++ b/src/common/backend/utils/resowner/resowner.cpp @@ -2548,4 +2548,13 @@ void ReleaseResownerOutOfTransaction() ResourceOwnerRelease(root, RESOURCE_RELEASE_BEFORE_LOCKS, false, true); ResourceOwnerRelease(root, RESOURCE_RELEASE_LOCKS, false, true); ResourceOwnerRelease(root, RESOURCE_RELEASE_AFTER_LOCKS, false, true); -} \ No newline at end of file +} + +FORCE_INLINE +Buffer ResourceOwnerGetBuffer(ResourceOwner owner) +{ + if (owner != NULL && owner->nbuffers > 0) { + return owner->buffers[owner->nbuffers - 1]; + } + return 0; +} diff --git a/src/gausskernel/cbb/instruments/gs_stack/gs_stack.cpp b/src/gausskernel/cbb/instruments/gs_stack/gs_stack.cpp index 304524577..669908126 100644 --- a/src/gausskernel/cbb/instruments/gs_stack/gs_stack.cpp +++ b/src/gausskernel/cbb/instruments/gs_stack/gs_stack.cpp @@ -371,9 +371,20 @@ void addr_to_name_reuse(void* pc, StringInfoData* call_stack) */ uint sym_off; if (dlinfo.dli_saddr && dlinfo.dli_sname) { - sym_off = (uint)((uintptr_t)pc - (uintptr_t)dlinfo.dli_saddr); - ereport(LOG, (errmsg("dlinfo.dli_sname %s.", dlinfo.dli_sname))); + if (ENABLE_DMS && (strcmp(dlinfo.dli_sname, "dms_request_page") == 0)) { + Buffer bufferid = ResourceOwnerGetBuffer(t_thrd.utils_cxt.CurrentResourceOwner); + if (bufferid > 0) { + BufferDesc *buf_desc = GetBufferDescriptor(bufferid - 1); + ereport(LOG, (errmsg("dlinfo.dli_sname %s. Current hold buffer: %u/%u/%u/%d %d-%u", + dlinfo.dli_sname, buf_desc->tag.rnode.spcNode, buf_desc->tag.rnode.dbNode, + buf_desc->tag.rnode.relNode, buf_desc->tag.rnode.bucketNode, buf_desc->tag.forkNum, + buf_desc->tag.blockNum))); + } + } else { + ereport(LOG, (errmsg("dlinfo.dli_sname %s.", dlinfo.dli_sname))); + } demangle_one_symbol(dlinfo.dli_sname, tmp_buf, sizeof(tmp_buf)); + sym_off = (uint)((uintptr_t)pc - (uintptr_t)dlinfo.dli_saddr); appendStringInfo(call_stack, "%s + 0x%x\n", tmp_buf, sym_off); } else { addr_to_name_bin(pc, dlinfo, call_stack); diff --git a/src/gausskernel/storage/lmgr/proc.cpp b/src/gausskernel/storage/lmgr/proc.cpp index f2c6cac0c..8830f5863 100755 --- a/src/gausskernel/storage/lmgr/proc.cpp +++ b/src/gausskernel/storage/lmgr/proc.cpp @@ -2089,6 +2089,10 @@ int ProcSleep(LOCALLOCK* locallock, LockMethod lockMethodTable, bool allow_con_u ereport(ERROR, (errcode(ERRCODE_LOCK_NOT_AVAILABLE), errmsg("could not obtain lock on row in relation,waitSec = %d", waitSec))); } else { + StringInfoData callStack; + initStringInfo(&callStack); + get_stack_according_to_tid(t_thrd.storage_cxt.conflicting_lock_thread_id, &callStack); + FreeStringInfo(&callStack); ereport(ERROR, (errcode(ERRCODE_LOCK_WAIT_TIMEOUT), (errmsg("Lock wait timeout: thread %lu on node %s waiting for %s on %s after %ld.%03d ms", t_thrd.proc_cxt.MyProcPid, g_instance.attr.attr_common.PGXCNodeName, modename, buf.data, msecs, diff --git a/src/gausskernel/storage/replication/logical/logicalfuncs.cpp b/src/gausskernel/storage/replication/logical/logicalfuncs.cpp index 97d3f9b29..4724933fd 100755 --- a/src/gausskernel/storage/replication/logical/logicalfuncs.cpp +++ b/src/gausskernel/storage/replication/logical/logicalfuncs.cpp @@ -493,7 +493,7 @@ static Datum pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool conf XLogRecord *record = NULL; char *errm = NULL; - record = XLogReadRecord(ctx->reader, startptr, &errm); + record = XLogReadRecord(ctx->reader, startptr, &errm, true, SS_XLOGDIR); if (errm != NULL) ereport(ERROR, (errcode(ERRCODE_LOGICAL_DECODE_ERROR), errmsg("Stopped to parse any valid XLog Record at %X/%X: %s.", diff --git a/src/include/utils/resowner.h b/src/include/utils/resowner.h index 4b309ea30..cd949eb18 100755 --- a/src/include/utils/resowner.h +++ b/src/include/utils/resowner.h @@ -210,4 +210,5 @@ extern void ResourceOwnerReleaseGlobalDBEntry(ResourceOwner owner, bool isCommit extern void ResourceOwnerReleaseGlobalIsExclusive(ResourceOwner owner, bool isCommit); extern bool CurrentResourceOwnerIsEmpty(ResourceOwner owner); extern void ReleaseResownerOutOfTransaction(); +extern Buffer ResourceOwnerGetBuffer(ResourceOwner owner); #endif /* RESOWNER_H */ From 8b0672c7bba0b4e8e7dac584027afb8cfb9006a8 Mon Sep 17 00:00:00 2001 From: li-qinlang Date: Mon, 14 Aug 2023 20:59:38 +0800 Subject: [PATCH 140/304] =?UTF-8?q?pldebugger=E6=94=AF=E6=8C=81=E5=8C=BF?= =?UTF-8?q?=E5=90=8D=E5=9D=97=E8=B0=83=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/nodes/outfuncs.cpp | 11 ++ src/common/backend/nodes/readfuncs.cpp | 12 ++ src/common/pl/plpgsql/src/pl_debugger.cpp | 121 +++++++++++++++++- src/common/pl/plpgsql/src/pl_debugger_api.cpp | 117 +++++++++++++---- src/common/pl/plpgsql/src/pl_handler.cpp | 23 ++++ src/include/nodes/nodes.h | 1 + src/include/utils/plpgsql.h | 15 +++ .../regress/expected/pl_debugger_client.out | 106 +++++++++++++++ .../regress/expected/pl_debugger_server.out | 34 ++++- src/test/regress/sql/pl_debugger_client.sql | 28 ++++ src/test/regress/sql/pl_debugger_server.sql | 33 +++++ 11 files changed, 471 insertions(+), 30 deletions(-) diff --git a/src/common/backend/nodes/outfuncs.cpp b/src/common/backend/nodes/outfuncs.cpp index 89d89a3af..193aa86aa 100755 --- a/src/common/backend/nodes/outfuncs.cpp +++ b/src/common/backend/nodes/outfuncs.cpp @@ -4275,6 +4275,14 @@ static void _outPLDebug_frame(StringInfo str, PLDebug_frame* node) WRITE_INT_FIELD(funcoid); } +static void _outPLDebug_codeline(StringInfo str, PLDebug_codeline* node) +{ + WRITE_NODE_TYPE("PLDEBUG_CODELINE"); + WRITE_INT_FIELD(lineno); + WRITE_STRING_FIELD(code); + WRITE_BOOL_FIELD(canBreak); +} + /* * @Description: Write hint node to string. * @out str: String buf. @@ -6967,6 +6975,9 @@ static void _outNode(StringInfo str, const void* obj) case T_PLDebug_frame: _outPLDebug_frame(str, (PLDebug_frame*) obj); break; + case T_PLDebug_codeline: + _outPLDebug_codeline(str, (PLDebug_codeline*) obj); + break; case T_CharsetCollateOptions: _outCharsetcollateOptions(str, (CharsetCollateOptions*)obj); break; diff --git a/src/common/backend/nodes/readfuncs.cpp b/src/common/backend/nodes/readfuncs.cpp index 079453cd9..254724861 100755 --- a/src/common/backend/nodes/readfuncs.cpp +++ b/src/common/backend/nodes/readfuncs.cpp @@ -1731,6 +1731,16 @@ static PLDebug_frame* _readPLDebug_frame(void) READ_DONE(); } +static PLDebug_codeline* _readPLDebug_codeline(void) +{ + READ_LOCALS(PLDebug_codeline); + READ_INT_FIELD(lineno); + READ_STRING_FIELD(code); + READ_BOOL_FIELD(canBreak); + + READ_DONE(); +} + /* * _readWithCheckOption */ @@ -6653,6 +6663,8 @@ Node* parseNodeString(void) return_value = _readPLDebug_breakPoint(); } else if (MATCH("PLDEBUG_FRAME", 13)) { return_value = _readPLDebug_frame(); + } else if (MATCH("PLDEBUG_CODELINE", 16)) { + return_value = _readPLDebug_codeline(); } else if (MATCH("TdigestData", 11)) { return_value = _readTdigestData(); } else if (MATCH("AUTO_INCREMENT", 14)) { diff --git a/src/common/pl/plpgsql/src/pl_debugger.cpp b/src/common/pl/plpgsql/src/pl_debugger.cpp index f75f68ce7..d9d7269f7 100644 --- a/src/common/pl/plpgsql/src/pl_debugger.cpp +++ b/src/common/pl/plpgsql/src/pl_debugger.cpp @@ -66,6 +66,8 @@ static void debug_server_disable_breakpoint(DebugInfo* debug); static void debug_server_info_breakpoint(DebugInfo* debug); static void debug_server_backtrace(); static void debug_server_set_variable(DebugInfo* debug, PLpgSQL_execstate* estate); +static void debug_server_info_code(DebugInfo* debug); +static char* debug_server_add_breakpoint_invalid(DebugInfo* debug, bool* valid, int lineno, int* newIndex, char* query, char* maskquery); /* close each debug function's resource if necessary */ void PlDebugerCleanUp(int code, Datum arg) @@ -138,8 +140,6 @@ static void init_debug_server(PLpgSQL_function* func, int socketId, int debugSta void check_debug(PLpgSQL_function* func, PLpgSQL_execstate* estate) { - if (func->fn_oid == InvalidOid) - return; bool found = false; bool need_continue_into = u_sess->plsql_cxt.cur_debug_server != NULL && ActiveBPInFunction(u_sess->plsql_cxt.cur_debug_server, func->fn_oid); @@ -339,6 +339,9 @@ bool handle_debug_msg(DebugInfo* debug, char* firstChar, PLpgSQL_execstate* esta case DEBUG_SET_VARIABLE_HEADER: debug_server_set_variable(debug, estate); break; + case DEBUG_INFOCODE_HEADER: + debug_server_info_code(debug); + break; default: ereport(ERROR, (errmodule(MOD_PLDEBUGGER), (errmsg("received unknown plsql debug client msg: %c", *firstChar)))); @@ -379,7 +382,12 @@ void server_send_end_msg(DebugInfo* debug) initStringInfo(&str); Oid funcoid = debug->func->fn_oid; Oid pkgoid = debug->func->pkg_oid; - char* funcname = get_func_name(debug->func->fn_oid); + char* funcname = NULL; + if (OidIsValid(funcoid)) { + funcname = get_func_name(funcoid); + } else { + funcname = pstrdup("anonymous block"); + } char* pkgname = NULL; Assert(funcname != NULL); if (pkgoid != InvalidOid) { @@ -877,7 +885,12 @@ static bool send_cur_info(DebugInfo* debug, PLpgSQL_execstate* estate, bool stop static bool get_cur_info(StringInfo str, PLpgSQL_execstate* estate, DebugInfo* debug) { Oid funcoid = debug->func->fn_oid; - char* funcname = get_func_name(funcoid); + char* funcname = NULL; + if (OidIsValid(funcoid)) { + funcname = get_func_name(funcoid); + } else { + funcname = pstrdup("anonymous block"); + } Oid pkgoid = debug->func->pkg_oid; Assert(funcname != NULL); int lineno = estate->err_stmt->lineno; @@ -1023,8 +1036,12 @@ static void debug_server_add_breakpoint(DebugInfo* debug) } int newIndex = -1; + bool valid = true; + if (funcOid == InvalidOid) { + query = debug_server_add_breakpoint_invalid(debug, &valid, lineno, &newIndex, query, maskquery); + } - if (!IsBreakPointExisted(debug, funcOid, lineno, false)) { + if (valid && !IsBreakPointExisted(debug, funcOid, lineno, false)) { PLDebug_breakPoint* bp = (PLDebug_breakPoint*)makeNode(PLDebug_breakPoint); bp->bpIndex = list_length(debug->bp_list); bp->funcoid = funcOid; @@ -1044,6 +1061,42 @@ static void debug_server_add_breakpoint(DebugInfo* debug) debug_server_send_msg(debug, str.data, str.len); } +static char* debug_server_add_breakpoint_invalid(DebugInfo* debug, bool* valid, int lineno, int* newIndex, char* query, char* maskquery) +{ + /* step 1: check whether exceed the max line number, return -2 if exceed */ + List* debug_body = debug->func->action->body; + uint32 maxline = ((PLpgSQL_stmt*)lfirst((list_head(debug_body))))->lineno + debug_body->length - 2; + *valid = (lineno < 1 || (uint32)lineno > maxline) ? false : true; + *newIndex = *valid ? -1 : ADD_BP_ERR_OUT_OF_RANGE; + /* step 2: valid breakpoint or not, return -3 if not valid */ + if (!*valid) { + return "NULL"; + } + List* breakables = collect_breakable_line(debug->func); + *valid = list_member_int(breakables, lineno); + *newIndex = *valid ? -1 : ADD_BP_ERR_INVALID_BP_POS; + list_free(breakables); + /*step 3: get query*/ + if (!*valid) { + return "NULL"; + } + ListCell *cell = NULL; + foreach(cell, debug_body) { + PLpgSQL_stmt* stmt = (PLpgSQL_stmt*)lfirst(cell); + if (stmt->lineno == lineno) { + query = pstrdup(get_stmt_query(stmt)); + maskquery = (query == NULL) ? NULL : maskPassword(query); + if (maskquery != NULL && maskquery != query) { + pfree_ext(query); + query = maskquery; + } + break; + } + } + return query; + /* not found? should not happen*/ +} + static void debug_server_delete_breakpoint(DebugInfo* debug) { /* Client guarantees the index to be positive */ @@ -1149,9 +1202,14 @@ PLDebug_frame* get_frame(DebugInfo* debug) erraction("Contact Huawei Engineer."))); } PLDebug_frame* frame = (PLDebug_frame*)makeNode(PLDebug_frame); - char* funcname = get_func_name(debug->func->fn_oid); + char* funcname = NULL; char* pkgname = NULL; Oid pkgoid = debug->func->pkg_oid; + if (OidIsValid(debug->func->fn_oid)) { + funcname = get_func_name(debug->func->fn_oid); + } else { + funcname = pstrdup("anonymous block"); + } if (pkgoid != InvalidOid) { NameData* pkgName = GetPackageName(pkgoid); pkgname = NameStr(*pkgName); @@ -1780,3 +1838,54 @@ void ReportInvalidMsg(const char* buf) errcause("Debugger send false messages."), erraction("Contact Huawei Engineer."))); } + +static void debug_server_info_code(DebugInfo* debug) +{ + List* infoCode = NIL; + DebugInfo* top = u_sess->plsql_cxt.cur_debug_server; + while (top != NULL) + { + if (!OidIsValid(top->func->fn_oid)) + { + break; + } + top = top->inner_called_debugger; + } + if (top == NULL) { + ereport(ERROR, (errmodule(MOD_PLDEBUGGER), errmsg("There is no anonymous block in debugging"))); + } + + List* debug_body = top->func->action->body; + List* breakables = collect_breakable_line(debug->func); + ListCell *cell = NULL; + + foreach(cell, debug_body) { + PLpgSQL_stmt* stmt = (PLpgSQL_stmt*)lfirst(cell); + PLDebug_codeline* code = (PLDebug_codeline*)makeNode(PLDebug_codeline);; + code->lineno = stmt->lineno; + code->code = pstrdup(get_stmt_query(stmt)); + code->canBreak = list_member_int(breakables, stmt->lineno); + infoCode = lappend(infoCode, code); + } + char* buf = nodeToString(infoCode); + int len = strlen(buf); + debug_server_send_msg(debug, buf, len); + list_free(breakables); + + cell = list_head(infoCode); + while (cell != NULL) { + ListCell* tmp = cell; + char* tmp_code = ((PLDebug_codeline*)lfirst(tmp))->code; + cell = lnext(cell); + if (tmp_code != NULL) { + pfree(tmp_code); + } + pfree(lfirst(tmp)); + pfree(tmp); + } + if (infoCode != NULL) { + pfree(infoCode); + } + + pfree(buf); +} \ No newline at end of file diff --git a/src/common/pl/plpgsql/src/pl_debugger_api.cpp b/src/common/pl/plpgsql/src/pl_debugger_api.cpp index d5c486116..8e7c0d5ad 100644 --- a/src/common/pl/plpgsql/src/pl_debugger_api.cpp +++ b/src/common/pl/plpgsql/src/pl_debugger_api.cpp @@ -66,10 +66,11 @@ static CodeLine* debug_show_code_worker(Oid funcid, uint32* num, int* headerline static void* debug_client_split_breakpoints_msg(uint32* num); static void* debug_client_split_localvariables_msg(uint32 *num); static void* debug_client_split_backtrace_msg(uint32* num); +static void* debug_client_split_infocode_msg(uint32* num); +static void debug_client_match_infocode_msg(Datum values[], bool nulls[], FuncCallContext *funcctx); static List* collect_breakable_line_oid(Oid funcOid); static void init_pldebug_htcl(); static bool CheckPlpgsqlFunc(Oid funcoid, bool report_error = true); -static List* collect_breakable_line(PLpgSQL_function* func); static Datum get_tuple_lineno_and_query(DebugClientInfo* client) { @@ -361,7 +362,21 @@ Datum debug_client_info_code(PG_FUNCTION_ARGS) funcctx->tuple_desc = BlessTupleDesc(tupdesc); int headerlines = 0; /* total number of tuples to be returned */ - funcctx->user_fctx = debug_show_code_worker(funcid, &(funcctx->max_calls), &headerlines); + if (OidIsValid(funcid)) { + funcctx->user_fctx = debug_show_code_worker(funcid, &(funcctx->max_calls), &headerlines); + } else { + DebugClientInfo* client = u_sess->plsql_cxt.debug_client; + if (client == NULL) { + ereport(ERROR, (errmodule(MOD_PLDEBUGGER), errmsg("There is no anonymous block in debugging"))); + } + StringInfoData str; + initStringInfo(&str); + appendStringInfo(&str, "0"); + debug_client_send_msg(client, DEBUG_INFOCODE_HEADER, str.data, str.len); + debug_client_rec_msg(client); + funcctx->user_fctx = debug_client_split_infocode_msg(&(funcctx->max_calls)); + pfree(str.data); + } (void)MemoryContextSwitchTo(oldcontext); } @@ -467,38 +482,55 @@ Datum debug_client_add_breakpoint(PG_FUNCTION_ARGS) Oid funcOid = PG_GETARG_OID(0); int32 lineno = PG_GETARG_INT32(1); - (void)CheckPlpgsqlFunc(funcOid); uint32 nLine = 0; int headerlines = 0; - CodeLine* lines = debug_show_code_worker(funcOid, &nLine, &headerlines); - - if (lineno < 1 || (uint32)lineno > nLine - headerlines) { - ereport(WARNING, (errcode(ERRCODE_WARNING), - errmsg("lineno must be within the range of [1, MaxLineNumber]" - " Please use dbe_pldebugger.info_code for valid breakpoint candidates"))); - PG_RETURN_INT32(-1); - } + CodeLine* lines = NULL; + CodeLine cl; + cl.code = NULL; + + if (OidIsValid(funcOid)) { + (void)CheckPlpgsqlFunc(funcOid); + lines = debug_show_code_worker(funcOid, &nLine, &headerlines); + if (lineno < 1 || (uint32)lineno > nLine - headerlines) { + ereport(WARNING, (errcode(ERRCODE_WARNING), + errmsg("lineno must be within the range of [1, MaxLineNumber]" + " Please use dbe_pldebugger.info_code for valid breakpoint candidates"))); + PG_RETURN_INT32(-1); + } - CodeLine cl = lines[(uint32)headerlines + lineno - 1]; - if (!cl.canBreak) { - ereport(WARNING, (errcode(ERRCODE_WARNING), - errmsg("the given line number does not name a valid breakpoint." - " Please use dbe_pldebugger.info_code for valid breakpoint candidates"))); - PG_RETURN_INT32(-1); + cl = lines[(uint32)headerlines + lineno - 1]; + if (!cl.canBreak) { + ereport(WARNING, (errcode(ERRCODE_WARNING), + errmsg("the given line number does not name a valid breakpoint." + " Please use dbe_pldebugger.info_code for valid breakpoint candidates"))); + PG_RETURN_INT32(-1); + } } DebugClientInfo* client = u_sess->plsql_cxt.debug_client; StringInfoData str; initStringInfo(&str); - appendStringInfo(&str, "%u:%d:%s", funcOid, lineno, cl.code); + appendStringInfo(&str, "%u:%d:%s", funcOid, lineno, cl.code == NULL ? "NULL" : cl.code); debug_client_send_msg(client, DEBUG_ADDBREAKPOINT_HEADER, str.data, str.len); debug_client_rec_msg(client); int32 ans = pg_strtoint32(client->rec_buffer); - pfree(lines); - if (ans == -1) { + pfree_ext(lines); + + if (ans == ADD_BP_ERR_ALREADY_EXISTS) { ereport(WARNING, (errcode(ERRCODE_WARNING), errmsg("the given line number already contains a valid breakpoint." " Please se dbe_pldebugger.info_breakpoints for detail."))); + PG_RETURN_INT32(-1); + } else if (ans == ADD_BP_ERR_OUT_OF_RANGE) { + ereport(WARNING, (errcode(ERRCODE_WARNING), + errmsg("lineno must be within the range of [1, MaxLineNumber]" + " Please use dbe_pldebugger.info_code for valid breakpoint candidates"))); + PG_RETURN_INT32(-1); + } else if (ans == ADD_BP_ERR_INVALID_BP_POS) { + ereport(WARNING, (errcode(ERRCODE_WARNING), + errmsg("the given line number does not name a valid breakpoint." + " Please use dbe_pldebugger.info_code for valid breakpoint candidates"))); + PG_RETURN_INT32(-1); } PG_RETURN_INT32(ans); } @@ -1038,8 +1070,9 @@ static PlDebugEntry* add_debug_func(Oid key) init_pldebug_htcl(); } - (void)CheckPlpgsqlFunc(key); - + if (OidIsValid(key)) { + (void)CheckPlpgsqlFunc(key); + } bool found = false; int commIdx = GetValidDebugCommIdx(); if (commIdx == -1) { @@ -1235,6 +1268,44 @@ static List* collect_breakable_line_oid(Oid funcOid) return collect_breakable_line(func); } +static void* debug_client_split_infocode_msg(uint32* num) +{ + char* msg = u_sess->plsql_cxt.debug_client->rec_buffer; + Node* ret = (Node*)stringToNode(msg); + if (ret == NULL || !(IsA(ret, List))) { + *num = 0; + return NULL; + } + + List* list = (List*)ret; + ListCell* lc = NULL; + Size length = list_length(list); + Size array_size = mul_size(sizeof(CodeLine), length); + CodeLine* codes = (CodeLine*)palloc0(array_size); + CodeLine* code = NULL; + int index = 0; + + foreach(lc, list) { + Node* n = (Node*)lfirst(lc); + if (!IsA(n, PLDebug_codeline)) { + goto error; + } + code = codes + index; + PLDebug_codeline* c = (PLDebug_codeline*)n; + code->lineno = c->lineno; + code->code = pstrdup(c->code); + code->canBreak = c->canBreak; + index++; + } + *num = length; + return codes; + +error: + ereport(DEBUG1, (errmodule(MOD_PLDEBUGGER), errmsg("False output for codeline type:\n%s", msg))); + ereport(ERROR, (errmodule(MOD_PLDEBUGGER), errmsg("Get unexpected output for codeline type."))); + return NULL; +} + CodeLine* debug_show_code_worker(Oid funcid, uint32* num, int* headerlines) { /* Get the raw results */ @@ -1469,7 +1540,7 @@ static void collect_breakable_line_walker(const List* stmts, List** lines) } } -static List* collect_breakable_line(PLpgSQL_function* func) +List* collect_breakable_line(PLpgSQL_function* func) { if (func == NULL) { return NIL; diff --git a/src/common/pl/plpgsql/src/pl_handler.cpp b/src/common/pl/plpgsql/src/pl_handler.cpp index 763f6c2dd..edc8d9b14 100755 --- a/src/common/pl/plpgsql/src/pl_handler.cpp +++ b/src/common/pl/plpgsql/src/pl_handler.cpp @@ -1128,6 +1128,7 @@ Datum plpgsql_inline_handler(PG_FUNCTION_ARGS) save_compile_context = u_sess->plsql_cxt.curr_compile_context; int save_compile_list_length = list_length(u_sess->plsql_cxt.compile_context_list); int save_compile_status = u_sess->plsql_cxt.compile_status; + DebugInfo* save_debug_info = func->debug; FormatCallStack* saveplcallstack = t_thrd.log_cxt.call_stack; PG_TRY(); { @@ -1149,6 +1150,16 @@ Datum plpgsql_inline_handler(PG_FUNCTION_ARGS) DecreasePackageUseCount(func); #ifndef ENABLE_MULTIPLE_NODES + /* debug finished, close debug resource */ + if (func->debug) { + /* if debuger is waiting for end msg, send end */ + server_send_end_msg(func->debug); + /* pass opt to upper debug function */ + server_pass_upper_debug_opt(func->debug); + clean_up_debug_server(func->debug, false, true); + delete_debug_func(InvalidOid); + } + func->debug = save_debug_info; /* for restore parent session and automn session package var values */ (void)processAutonmSessionPkgsInException(func); @@ -1169,6 +1180,18 @@ Datum plpgsql_inline_handler(PG_FUNCTION_ARGS) PG_RE_THROW(); } PG_END_TRY(); +#ifndef ENABLE_MULTIPLE_NODES + /* debug finished, close debug resource */ + if (func->debug) { + /* if debuger is waiting for end msg, send end */ + server_send_end_msg(func->debug); + /* pass opt to upper debug function */ + server_pass_upper_debug_opt(func->debug); + clean_up_debug_server(func->debug, false, true); + delete_debug_func(InvalidOid); + } + func->debug = save_debug_info; +#endif if (u_sess->SPI_cxt._connected == 0) { t_thrd.utils_cxt.STPSavedResourceOwner = NULL; } diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index 19165533d..f6d307350 100755 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -818,6 +818,7 @@ typedef enum NodeTag { T_PLDebug_variable, T_PLDebug_breakPoint, T_PLDebug_frame, + T_PLDebug_codeline, T_TdigestData, diff --git a/src/include/utils/plpgsql.h b/src/include/utils/plpgsql.h index 1768581ef..35e914971 100644 --- a/src/include/utils/plpgsql.h +++ b/src/include/utils/plpgsql.h @@ -1497,6 +1497,19 @@ typedef struct PlDebugEntry { PLpgSQL_function* func; } PlDebugEntry; +typedef enum AddBreakPointError { + ADD_BP_ERR_ALREADY_EXISTS = -1, + ADD_BP_ERR_OUT_OF_RANGE = -2, + ADD_BP_ERR_INVALID_BP_POS = -3 +} AddBreakPointError; + +typedef struct PLDebug_codeline { + NodeTag type; + int lineno; + char* code; + bool canBreak; +} PLDebug_codeline; + typedef List* (*RawParserHook)(const char*, List**); const int MAXINT8LEN = 25; @@ -1527,6 +1540,7 @@ const char DEBUG_STEP_INTO_HEADER = 's'; const char DEBUG_STEP_INTO_HEADER_AFTER = 'S'; const char DEBUG_BACKTRACE_HEADER = 't'; const char DEBUG_SET_VARIABLE_HEADER = 'h'; +const char DEBUG_INFOCODE_HEADER = 'i'; /* server return message */ const int DEBUG_SERVER_SUCCESS = 0; @@ -1632,6 +1646,7 @@ extern void RecvUnixMsg(const char* buf, int bufLen, char* destBuf, int destLen) extern char* ResizeDebugBufferIfNecessary(char* buffer, int* oldSize, int needSize); extern void ReleaseDebugCommIdx(int idx); extern void SendUnixMsg(int socket, const char* val, int len, bool is_client); +extern List* collect_breakable_line(PLpgSQL_function* func); /********************************************************************** * Function declarations diff --git a/src/test/regress/expected/pl_debugger_client.out b/src/test/regress/expected/pl_debugger_client.out index ff334e80d..4c9d95ff2 100755 --- a/src/test/regress/expected/pl_debugger_client.out +++ b/src/test/regress/expected/pl_debugger_client.out @@ -1510,3 +1510,109 @@ SELECT * FROM DBE_PLDEBUGGER.continue(); --?.* (1 row) +--test anonymous +select pg_sleep(1); + pg_sleep +---------- + +(1 row) + +delete from tmp_holder; +select dbe_pldebugger.attach(nodename, port) from debug_info; + attach +------------------------------------------------------------------------------------------------------------------ + (0,"""anonymous block""",6," select oid from pg_proc into funcoid where proname='abs' and prosrc='int8abs';") +(1 row) + +select dbe_pldebugger.info_code(0); + info_code +-------------------------------------------------------------------------------------------- + (6," select oid from pg_proc into funcoid where proname='abs' and prosrc='int8abs';",t) + (7," perform * from pg_proc where oid = funcoid;",t) + (8," k = test_increment(3);",t) + (9," insert into test_anonymous values(k, 'test');",t) + (10," k = abs(-k);",t) + (11," insert into test_anonymous values(k, 'test2');",t) + (,,f) +(7 rows) + +insert into tmp_holder select * from dbe_pldebugger.add_breakpoint(0, 0); -- negative +WARNING: lineno must be within the range of [1, MaxLineNumber] Please use dbe_pldebugger.info_code for valid breakpoint candidates +insert into tmp_holder select * from dbe_pldebugger.add_breakpoint(0, 6); -- headerline +insert into tmp_holder select * from dbe_pldebugger.add_breakpoint(0, 6); -- already +WARNING: the given line number already contains a valid breakpoint. Please se dbe_pldebugger.info_breakpoints for detail. +insert into tmp_holder select * from dbe_pldebugger.add_breakpoint(0, 8); -- ok +insert into tmp_holder select * from dbe_pldebugger.add_breakpoint(0, 13); -- negative +WARNING: lineno must be within the range of [1, MaxLineNumber] Please use dbe_pldebugger.info_code for valid breakpoint candidates +insert into tmp_holder select * from dbe_pldebugger.disable_breakpoint(1); -- ok +insert into tmp_holder select * from dbe_pldebugger.enable_breakpoint(1); -- ok +insert into tmp_holder select * from dbe_pldebugger.delete_breakpoint(0); -- ok +select funcname, lineno, query from dbe_pldebugger.next(); + funcname | lineno | query +-------------------+--------+----------------------------------------------------- + "anonymous block" | 7 | perform * from pg_proc where oid = funcoid; +(1 row) + +select funcname, lineno, query from dbe_pldebugger.continue(); + funcname | lineno | query +-------------------+--------+---------------------------- + "anonymous block" | 8 | k = test_increment(3); +(1 row) + +select funcname, lineno, query from dbe_pldebugger.step(); + funcname | lineno | query +----------------+--------+----------------- + test_increment | 5 | x := x + 1; +(1 row) + +select funcname, lineno, query from dbe_pldebugger.next(); + funcname | lineno | query +----------------+--------+----------------- + test_increment | 6 | y := y * 2; +(1 row) + +select frameno, funcname, lineno, query from dbe_pldebugger.backtrace(); + frameno | funcname | lineno | query +---------+-------------------+--------+---------------------------- + 0 | test_increment | 6 | y := y * 2; + 1 | "anonymous block" | 8 | k = test_increment(3); +(2 rows) + +select * from dbe_pldebugger.info_locals(); + varname | vartype | value | package_name | isconst +---------+---------+-------+--------------+--------- + x | int4 | 4 | | f + y | int4 | 0 | | f +(2 rows) + +select funcname, lineno, query from dbe_pldebugger.finish(); + funcname | lineno | query +-------------------+--------+--------------------------------------------------- + "anonymous block" | 9 | insert into test_anonymous values(k, 'test'); +(1 row) + +select * from dbe_pldebugger.print_var('k'); + varname | vartype | value | package_name | isconst +---------+---------+-------+--------------+--------- + k | int4 | 0 | | f +(1 row) + +select * from dbe_pldebugger.abort(); + abort +------- + t +(1 row) + +select * from tmp_holder; + res +------ + -1 + 0 + -1 + 1 + -1 + true + true + true +(8 rows) + diff --git a/src/test/regress/expected/pl_debugger_server.out b/src/test/regress/expected/pl_debugger_server.out index 9f0c8d488..22686fbb4 100755 --- a/src/test/regress/expected/pl_debugger_server.out +++ b/src/test/regress/expected/pl_debugger_server.out @@ -655,8 +655,38 @@ select z_pk.pro1(1,2,'+'); 13 (1 row) +--test anonymous +delete from debug_info; +CREATE OR REPLACE FUNCTION test_increment(x int) RETURNS int AS +$BODY$ +DECLARE + y int; +BEGIN + x := x + 1; + y := y * 2; + RETURN y; +END +$BODY$ +LANGUAGE plpgsql; +CREATE TABLE test_anonymous(a int, b text); +insert into debug_info select * from dbe_pldebugger.turn_on(0); +do $$ +declare + funcoid oid; + k int; +begin + select oid from pg_proc into funcoid where proname='abs' and prosrc='int8abs'; + perform * from pg_proc where oid = funcoid; + k = test_increment(3); + insert into test_anonymous values(k, 'test'); + k = abs(-k); + insert into test_anonymous values(k, 'test2'); +end; +$$; +ERROR: receive abort message +CONTEXT: PL/pgSQL function inline_code_block line 9 at SQL statement drop schema pl_debugger cascade; -NOTICE: drop cascades to 18 other objects +NOTICE: drop cascades to 20 other objects DETAIL: drop cascades to table test drop cascades to function test_debug(integer) drop cascades to table show_code_table @@ -675,3 +705,5 @@ drop cascades to function test_setvar(integer) --?drop cascades to package .* drop cascades to function pl_debugger.pro1(integer,integer,character varying) drop cascades to function pl_debugger.pro2(integer,character varying) +drop cascades to function test_increment(integer) +drop cascades to table test_anonymous diff --git a/src/test/regress/sql/pl_debugger_client.sql b/src/test/regress/sql/pl_debugger_client.sql index e9efd11ff..aca8125d5 100644 --- a/src/test/regress/sql/pl_debugger_client.sql +++ b/src/test/regress/sql/pl_debugger_client.sql @@ -464,3 +464,31 @@ SELECT * FROM DBE_PLDEBUGGER.info_locals(); SELECT * FROM DBE_PLDEBUGGER.backtrace(); SELECT * FROM DBE_PLDEBUGGER.continue(); + +--test anonymous +select pg_sleep(1); +delete from tmp_holder; + +select dbe_pldebugger.attach(nodename, port) from debug_info; + +select dbe_pldebugger.info_code(0); + +insert into tmp_holder select * from dbe_pldebugger.add_breakpoint(0, 0); -- negative +insert into tmp_holder select * from dbe_pldebugger.add_breakpoint(0, 6); -- headerline +insert into tmp_holder select * from dbe_pldebugger.add_breakpoint(0, 6); -- already +insert into tmp_holder select * from dbe_pldebugger.add_breakpoint(0, 8); -- ok +insert into tmp_holder select * from dbe_pldebugger.add_breakpoint(0, 13); -- negative +insert into tmp_holder select * from dbe_pldebugger.disable_breakpoint(1); -- ok +insert into tmp_holder select * from dbe_pldebugger.enable_breakpoint(1); -- ok +insert into tmp_holder select * from dbe_pldebugger.delete_breakpoint(0); -- ok +select funcname, lineno, query from dbe_pldebugger.next(); +select funcname, lineno, query from dbe_pldebugger.continue(); +select funcname, lineno, query from dbe_pldebugger.step(); +select funcname, lineno, query from dbe_pldebugger.next(); +select frameno, funcname, lineno, query from dbe_pldebugger.backtrace(); +select * from dbe_pldebugger.info_locals(); +select funcname, lineno, query from dbe_pldebugger.finish(); +select * from dbe_pldebugger.print_var('k'); +select * from dbe_pldebugger.abort(); + +select * from tmp_holder; \ No newline at end of file diff --git a/src/test/regress/sql/pl_debugger_server.sql b/src/test/regress/sql/pl_debugger_server.sql index befdc6145..904342c52 100644 --- a/src/test/regress/sql/pl_debugger_server.sql +++ b/src/test/regress/sql/pl_debugger_server.sql @@ -411,4 +411,37 @@ select * from debug_info; select z_pk.pro1(1,2,'+'); +--test anonymous +delete from debug_info; + +CREATE OR REPLACE FUNCTION test_increment(x int) RETURNS int AS +$BODY$ +DECLARE + y int; +BEGIN + x := x + 1; + y := y * 2; + RETURN y; +END +$BODY$ +LANGUAGE plpgsql; + +CREATE TABLE test_anonymous(a int, b text); + +insert into debug_info select * from dbe_pldebugger.turn_on(0); + +do $$ +declare + funcoid oid; + k int; +begin + select oid from pg_proc into funcoid where proname='abs' and prosrc='int8abs'; + perform * from pg_proc where oid = funcoid; + k = test_increment(3); + insert into test_anonymous values(k, 'test'); + k = abs(-k); + insert into test_anonymous values(k, 'test2'); +end; +$$; + drop schema pl_debugger cascade; From 4f90514e937ce487a65104e00f8fe303c4640e1d Mon Sep 17 00:00:00 2001 From: liuheng Date: Thu, 17 Aug 2023 10:34:19 +0800 Subject: [PATCH 141/304] =?UTF-8?q?=E6=B7=BB=E5=8A=A0cbb=20commit=20id?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/ddes/ddes_commit_id | 1 + 1 file changed, 1 insertion(+) diff --git a/src/gausskernel/ddes/ddes_commit_id b/src/gausskernel/ddes/ddes_commit_id index 4677baf34..7c95953f9 100644 --- a/src/gausskernel/ddes/ddes_commit_id +++ b/src/gausskernel/ddes/ddes_commit_id @@ -1,2 +1,3 @@ dms_commit_id=b12ceed7bb0d3de049d8b4858180c77110ff6444 dss_commit_id=0b9c9bc1a2c3cac463ea70fd6abc1790e9f125aa +cbb_commit_id=d45b3bdee26012a8d14adf1f536a2d8a24a177fe From b95efca6ef3d934e550e2c88ad54b52dd36aa719 Mon Sep 17 00:00:00 2001 From: chenxiaobin19 <1025221611@qq.com> Date: Tue, 15 Aug 2023 10:45:36 +0800 Subject: [PATCH 142/304] =?UTF-8?q?=E5=8F=91=E5=B8=83=E8=AE=A2=E9=98=85?= =?UTF-8?q?=E6=94=AF=E6=8C=81=E5=86=B2=E7=AA=81=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- contrib/test_decoding/test_decoding.cpp | 80 ----- doc/src/sgml/ref/alter_subscription.sgmlin | 11 + src/bin/gs_guc/cluster_guc.conf | 1 + .../backend/catalog/pg_subscription.cpp | 12 +- src/common/backend/parser/gram.y | 10 + .../backend/utils/misc/guc/guc_storage.cpp | 19 + .../optimizer/commands/subscriptioncmds.cpp | 340 ++++++++++-------- .../runtime/executor/execReplication.cpp | 49 ++- .../replication/logical/parallel_decode.cpp | 18 +- .../storage/replication/logical/worker.cpp | 288 ++++++++++++++- src/include/catalog/pg_subscription.h | 9 +- .../knl/knl_guc/knl_session_attr_storage.h | 1 + src/include/knl/knl_thread.h | 1 + src/include/replication/logical.h | 2 + src/include/replication/replicainternal.h | 7 + src/test/regress/input/subscription.source | 8 + .../regress/output/recovery_2pc_tools.source | 1 + src/test/regress/output/subscription.source | 25 +- src/test/subscription/schedule | 4 +- src/test/subscription/testcase/disable.sh | 65 ++++ src/test/subscription/testcase/skiplsn.sh | 69 ++++ 21 files changed, 767 insertions(+), 253 deletions(-) create mode 100644 src/test/subscription/testcase/disable.sh create mode 100644 src/test/subscription/testcase/skiplsn.sh diff --git a/contrib/test_decoding/test_decoding.cpp b/contrib/test_decoding/test_decoding.cpp index 8f5b377b7..76175dd9b 100644 --- a/contrib/test_decoding/test_decoding.cpp +++ b/contrib/test_decoding/test_decoding.cpp @@ -199,87 +199,7 @@ static bool pg_decode_filter(LogicalDecodingContext* ctx, RepOriginId origin_id) return true; return false; } -static void tuple_to_stringinfo(Relation relation, StringInfo s, TupleDesc tupdesc, HeapTuple tuple, bool isOld) -{ - if ((tuple->tupTableType == HEAP_TUPLE) && (HEAP_TUPLE_IS_COMPRESSED(tuple->t_data) || - (int)HeapTupleHeaderGetNatts(tuple->t_data, tupdesc) > tupdesc->natts)) { - return; - } - - Oid oid; - /* print oid of tuple, it's not included in the TupleDesc */ - if ((oid = HeapTupleHeaderGetOid(tuple->t_data)) != InvalidOid) { - appendStringInfo(s, " oid[oid]:%u", oid); - } - - /* print all columns individually */ - for (int natt = 0; natt < tupdesc->natts; natt++) { - Form_pg_attribute attr; /* the attribute itself */ - Oid typoutput; /* output function */ - bool typisvarlena = false; - Datum origval; /* possibly toasted Datum */ - bool isnull = true; /* column is null? */ - - attr = &tupdesc->attrs[natt]; - - /* - * don't print dropped columns, we can't be sure everything is - * available for them - */ - if (attr->attisdropped) - continue; - - /* - * Don't print system columns, oid will already have been printed if - * present. - */ - if (attr->attnum < 0 || (isOld && !IsRelationReplidentKey(relation, attr->attnum))) - continue; - - Oid typid = attr->atttypid; /* type of current attribute */ - - /* get Datum from tuple */ - if (tuple->tupTableType == HEAP_TUPLE) { - origval = heap_getattr(tuple, natt + 1, tupdesc, &isnull); - } else { - origval = uheap_getattr((UHeapTuple)tuple, natt + 1, tupdesc, &isnull); - } - - /* print attribute name */ - appendStringInfoChar(s, ' '); - appendStringInfoString(s, quote_identifier(NameStr(attr->attname))); - - /* print attribute type */ - appendStringInfoChar(s, '['); - char* type_name = format_type_be(typid); - if (strlen(type_name) == strlen("clob") && strncmp(type_name, "clob", strlen("clob")) == 0) { - errno_t rc = strcpy_s(type_name, sizeof("clob"), "text"); - securec_check_c(rc, "\0", "\0"); - } - appendStringInfoString(s, type_name); - appendStringInfoChar(s, ']'); - - /* query output function */ - getTypeOutputInfo(typid, &typoutput, &typisvarlena); - - /* print separator */ - appendStringInfoChar(s, ':'); - - /* print data */ - if (isnull) - appendStringInfoString(s, "null"); - else if (typisvarlena && VARATT_IS_EXTERNAL_ONDISK_B(origval)) - appendStringInfoString(s, "unchanged-toast-datum"); - else if (!typisvarlena) - PrintLiteral(s, typid, OidOutputFunctionCall(typoutput, origval)); - else { - Datum val; /* definitely detoasted Datum */ - val = PointerGetDatum(PG_DETOAST_DATUM(origval)); - PrintLiteral(s, typid, OidOutputFunctionCall(typoutput, val)); - } - } -} /* * callback for individual changed tuples */ diff --git a/doc/src/sgml/ref/alter_subscription.sgmlin b/doc/src/sgml/ref/alter_subscription.sgmlin index 2c4e4d168..6ad2fe06c 100644 --- a/doc/src/sgml/ref/alter_subscription.sgmlin +++ b/doc/src/sgml/ref/alter_subscription.sgmlin @@ -25,6 +25,7 @@ ALTER SUBSCRIPTION name CONNECTION ALTER SUBSCRIPTION name SET PUBLICATION publication_name [, ...] ALTER SUBSCRIPTION name REFRESH PUBLICATION [ WITH ( refresh_option [= value] [, ... ] ) ] ALTER SUBSCRIPTION name ENABLE +ALTER SUBSCRIPTION name DISABLE ALTER SUBSCRIPTION name SET ( subscription_parameter [= value] [, ... ] ) ALTER SUBSCRIPTION name OWNER TO new_owner ALTER SUBSCRIPTION name RENAME TO new_name @@ -134,6 +135,16 @@ ALTER SUBSCRIPTION name RENAME TO < + + DISABLE + + + Disables a running subscription, stopping the logical replication + worker at the end of the transaction. + + + + SET ( subscription_parameter [= value] [, ... ] ) diff --git a/src/bin/gs_guc/cluster_guc.conf b/src/bin/gs_guc/cluster_guc.conf index ae2c84f92..26adb9f44 100755 --- a/src/bin/gs_guc/cluster_guc.conf +++ b/src/bin/gs_guc/cluster_guc.conf @@ -751,6 +751,7 @@ light_comm|bool|0,0|NULL|NULL| ignore_standby_lsn_window|int|0,2147483647|ms|NULL| ignore_feedback_xmin_window|int|0,2147483647|ms|NULL| ss_enable_bcast_snapshot|bool|0,0|NULL|NULL| +subscription_conflict_resolution|enum|error,apply_remote,keep_local|NULL|NULL| [cmserver] log_dir|string|0,0|NULL|NULL| log_file_size|int|0,2047|MB|NULL| diff --git a/src/common/backend/catalog/pg_subscription.cpp b/src/common/backend/catalog/pg_subscription.cpp index 86b7dea51..5f075ff2d 100644 --- a/src/common/backend/catalog/pg_subscription.cpp +++ b/src/common/backend/catalog/pg_subscription.cpp @@ -103,6 +103,14 @@ Subscription *GetSubscription(Oid subid, bool missing_ok) sub->binary = DatumGetBool(datum); } + /* Get skiplsn */ + datum = SysCacheGetAttr(SUBSCRIPTIONOID, tup, Anum_pg_subscription_subskiplsn, &isnull); + if (unlikely(isnull)) { + sub->skiplsn = InvalidXLogRecPtr; + } else { + sub->skiplsn = TextDatumGetLsn(datum); + } + ReleaseSysCache(tup); return sub; @@ -237,7 +245,7 @@ static List *textarray_to_stringlist(ArrayType *textarray) return res; } -static Datum LsnGetTextDatum(XLogRecPtr lsn) +Datum LsnGetTextDatum(XLogRecPtr lsn) { char clsn[MAXFNAMELEN]; int ret = snprintf_s(clsn, sizeof(clsn), sizeof(clsn) - 1, "%X/%X", (uint32)(lsn >> 32), (uint32)lsn); @@ -246,7 +254,7 @@ static Datum LsnGetTextDatum(XLogRecPtr lsn) return CStringGetTextDatum(clsn); } -static XLogRecPtr TextDatumGetLsn(Datum datum) +XLogRecPtr TextDatumGetLsn(Datum datum) { XLogRecPtr lsn; uint32 lsn_hi; diff --git a/src/common/backend/parser/gram.y b/src/common/backend/parser/gram.y index 17d8545e6..332202531 100644 --- a/src/common/backend/parser/gram.y +++ b/src/common/backend/parser/gram.y @@ -18854,6 +18854,16 @@ AlterSubscriptionStmt: n->options = list_make1(makeDefElem("enabled", (Node *)makeInteger(TRUE))); $$ = (Node *)n; + } + | ALTER SUBSCRIPTION name DISABLE_P + { + AlterSubscriptionStmt *n = + makeNode(AlterSubscriptionStmt); + n->refresh = false; + n->subname = $3; + n->options = list_make1(makeDefElem("enabled", + (Node *)makeInteger(FALSE))); + $$ = (Node *)n; } ; /***************************************************************************** diff --git a/src/common/backend/utils/misc/guc/guc_storage.cpp b/src/common/backend/utils/misc/guc/guc_storage.cpp index f3692badf..7d22f6c84 100755 --- a/src/common/backend/utils/misc/guc/guc_storage.cpp +++ b/src/common/backend/utils/misc/guc/guc_storage.cpp @@ -293,6 +293,13 @@ static const struct config_enum_entry repl_auth_mode_options[] = { {NULL, 0, false} }; +static const struct config_enum_entry ConflictResolvers[] = { + {"error", RESOLVE_ERROR, false}, + {"apply_remote", RESOLVE_APPLY_REMOTE, false}, + {"keep_local", RESOLVE_KEEP_LOCAL, false}, + {NULL, 0, false} +}; + /* * Although only "on", "off", "remote_write", and "local" are documented, we * accept all the likely variants of "on" and "off". @@ -4723,6 +4730,18 @@ static void InitStorageConfigureNamesEnum() NULL, NULL, NULL}, + {{"subscription_conflict_resolution", + PGC_SIGHUP, + NODE_SINGLENODE, + REPLICATION, + gettext_noop("Sets method used for conflict resolution for resolvable conflicts."), + NULL}, + &u_sess->attr.attr_storage.subscription_conflict_resolution, + RESOLVE_ERROR, + ConflictResolvers, + NULL, + NULL, + NULL}, #ifndef ENABLE_MULTIPLE_NODES {{"dcf_log_file_permission", PGC_POSTMASTER, diff --git a/src/gausskernel/optimizer/commands/subscriptioncmds.cpp b/src/gausskernel/optimizer/commands/subscriptioncmds.cpp index eea9de4dc..aa52f2ebe 100644 --- a/src/gausskernel/optimizer/commands/subscriptioncmds.cpp +++ b/src/gausskernel/optimizer/commands/subscriptioncmds.cpp @@ -45,10 +45,46 @@ #include "utils/syscache.h" #include "utils/array.h" #include "utils/acl.h" +#include "utils/pg_lsn.h" #include "access/tableam.h" #include "libpq/libpq-fe.h" #include "replication/slot.h" +/* + * Options that can be specified by the user in CREATE/ALTER SUBSCRIPTION + * command. + */ +#define SUBOPT_CONNINFO 0x00000001 +#define SUBOPT_PUBLICATION 0x00000002 +#define SUBOPT_ENABLED 0x00000004 +#define SUBOPT_SLOT_NAME 0x00000008 +#define SUBOPT_SYNCHRONOUS_COMMIT 0x00000010 +#define SUBOPT_BINARY 0x00000020 +#define SUBOPT_COPY_DATA 0x00000040 +#define SUBOPT_CONNECT 0x00000080 +#define SUBOPT_SKIPLSN 0x00000100 + +/* check if the 'val' has 'bits' set */ +#define IsSet(val, bits) (((val) & (bits)) == (bits)) + +/* + * Structure to hold a bitmap representing the user-provided CREATE/ALTER + * SUBSCRIPTION command options and the parsed/default values of each of them. + */ +typedef struct SubOpts +{ + bits32 specified_opts; + char *conninfo; + List *publications; + bool enabled; + char *slot_name; + char *synchronous_commit; + bool binary; + bool copy_data; + bool connect; + XLogRecPtr skiplsn; +} SubOpts; + static bool ConnectPublisher(char* conninfo, char* slotname); static void CreateSlotInPublisherAndInsertSubRel(char *slotname, Oid subid, List *publications, bool *copy_data, bool create_slot); @@ -61,120 +97,127 @@ static bool CheckPublicationsExistOnPublisher(List *publications); * Common option parsing function for CREATE and ALTER SUBSCRIPTION commands. * * Since not all options can be specified in both commands, this function - * will report an error on options if the target output pointer is NULL to - * accommodate that. + * will report an error mutually exclusive options are specified. + * + * Caller is expected to have cleared 'opts'. */ -static void parse_subscription_options(const List *options, char **conninfo, List **publications, bool *enabled_given, - bool *enabled, bool *slot_name_given, char **slot_name, char **synchronous_commit, bool *binary_given, bool *binary, - bool *copy_data_given, bool *copy_data, bool *connect_given, bool *connect) +static void parse_subscription_options(const List *stmt_options, bits32 supported_opts, SubOpts *opts) { ListCell *lc; - if (conninfo) { - *conninfo = NULL; - } - if (publications) { - *publications = NIL; - } - if (enabled) { - *enabled_given = false; - } - if (slot_name) { - *slot_name_given = false; - *slot_name = NULL; - } - if (synchronous_commit) { - *synchronous_commit = NULL; - } - if (binary) { - *binary_given = false; - *binary = false; - } + /* caller must expect some option */ + Assert(supported_opts != 0); - if (copy_data) { - *copy_data_given = false; - *copy_data = true; + /* Set default values for the boolean supported options. */ + if (IsSet(supported_opts, SUBOPT_BINARY)) { + opts->binary = false; } - - if (connect) { - *connect_given = false; - *connect = true; + if (IsSet(supported_opts, SUBOPT_COPY_DATA)) { + opts->copy_data = true; + } + if (IsSet(supported_opts, SUBOPT_CONNECT)) { + opts->connect = true; } /* Parse options */ - foreach (lc, options) { + foreach (lc, stmt_options) { DefElem *defel = (DefElem *)lfirst(lc); - if (strcmp(defel->defname, "conninfo") == 0 && conninfo) { - if (*conninfo) { + if (IsSet(supported_opts, SUBOPT_CONNINFO) && strcmp(defel->defname, "conninfo") == 0) { + if (IsSet(opts->specified_opts, SUBOPT_CONNINFO)) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("conflicting or redundant options"))); } - *conninfo = defGetString(defel); - } else if (strcmp(defel->defname, "publication") == 0 && publications) { - if (*publications) { + opts->specified_opts |= SUBOPT_CONNINFO; + opts->conninfo = defGetString(defel); + } else if (IsSet(supported_opts, SUBOPT_PUBLICATION) && strcmp(defel->defname, "publication") == 0) { + if (IsSet(opts->specified_opts, SUBOPT_PUBLICATION)) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("conflicting or redundant options"))); } - *publications = defGetStringList(defel); - } else if (strcmp(defel->defname, "enabled") == 0 && enabled) { - if (*enabled_given) { + opts->specified_opts |= SUBOPT_PUBLICATION; + opts->publications = defGetStringList(defel); + } else if (IsSet(supported_opts, SUBOPT_ENABLED) && strcmp(defel->defname, "enabled") == 0) { + if (IsSet(opts->specified_opts, SUBOPT_ENABLED)) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("conflicting or redundant options"))); } - *enabled_given = true; - *enabled = defGetBoolean(defel); - } else if (strcmp(defel->defname, "slot_name") == 0 && slot_name) { - if (*slot_name_given) { + opts->specified_opts |= SUBOPT_ENABLED; + opts->enabled = defGetBoolean(defel); + } else if (IsSet(supported_opts, SUBOPT_SLOT_NAME) && strcmp(defel->defname, "slot_name") == 0) { + if (IsSet(opts->specified_opts, SUBOPT_SLOT_NAME)) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("conflicting or redundant options"))); } - *slot_name_given = true; - *slot_name = defGetString(defel); + opts->specified_opts |= SUBOPT_SLOT_NAME; + opts->slot_name = defGetString(defel); /* Setting slot_name = NONE is treated as no slot name. */ - if (strcmp(*slot_name, "none") == 0) { - *slot_name = NULL; + if (strcmp(opts->slot_name, "none") == 0) { + opts->slot_name = NULL; } else { - ReplicationSlotValidateName(*slot_name, ERROR); + ReplicationSlotValidateName(opts->slot_name, ERROR); } - } else if (strcmp(defel->defname, "synchronous_commit") == 0 && synchronous_commit) { - if (*synchronous_commit) { + } else if (IsSet(supported_opts, SUBOPT_SYNCHRONOUS_COMMIT) && + strcmp(defel->defname, "synchronous_commit") == 0) { + if (IsSet(opts->specified_opts, SUBOPT_SYNCHRONOUS_COMMIT)) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("conflicting or redundant options"))); } - *synchronous_commit = defGetString(defel); + opts->specified_opts |= SUBOPT_SYNCHRONOUS_COMMIT; + opts->synchronous_commit = defGetString(defel); /* Test if the given value is valid for synchronous_commit GUC. */ - (void)set_config_option("synchronous_commit", *synchronous_commit, PGC_BACKEND, PGC_S_TEST, GUC_ACTION_SET, - false, 0, false); - } else if (strcmp(defel->defname, "binary") == 0 && binary) { - if (*binary_given) { + (void)set_config_option("synchronous_commit", opts->synchronous_commit, PGC_BACKEND, PGC_S_TEST, + GUC_ACTION_SET, false, 0, false); + } else if (IsSet(supported_opts, SUBOPT_BINARY) && strcmp(defel->defname, "binary") == 0) { + if (IsSet(opts->specified_opts, SUBOPT_BINARY)) { + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("conflicting or redundant options"))); + } + + opts->specified_opts |= SUBOPT_BINARY; + opts->binary = defGetBoolean(defel); + } else if (IsSet(supported_opts, SUBOPT_COPY_DATA) && strcmp(defel->defname, "copy_data") == 0) { + if (IsSet(opts->specified_opts, SUBOPT_COPY_DATA)) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("conflicting or redundant options"))); } - *binary_given = true; - *binary = defGetBoolean(defel); - } else if (strcmp(defel->defname, "copy_data") == 0 && copy_data) { - if (*copy_data_given) { + opts->specified_opts |= SUBOPT_COPY_DATA; + opts->copy_data = defGetBoolean(defel); + } else if (IsSet(supported_opts, SUBOPT_CONNECT) && strcmp(defel->defname, "connect") == 0) { + if (IsSet(opts->specified_opts, SUBOPT_CONNECT)) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("conflicting or redundant options"))); } - *copy_data_given = true; - *copy_data = defGetBoolean(defel); - } else if (strcmp(defel->defname, "connect") == 0 && connect) { - if (*connect_given) { + opts->specified_opts = SUBOPT_CONNECT; + opts->connect = defGetBoolean(defel); + } else if (IsSet(supported_opts, SUBOPT_SKIPLSN) && strcmp(defel->defname, "skiplsn") == 0) { + if (IsSet(opts->specified_opts, SUBOPT_SKIPLSN)) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("conflicting or redundant options"))); } - *connect_given = true; - *connect = defGetBoolean(defel); + char *lsn_str = defGetString(defel); + opts->specified_opts = SUBOPT_SKIPLSN; + /* Setting lsn = 'NONE' is treated as resetting LSN */ + if (strcmp(lsn_str, "none") == 0) { + opts->skiplsn = InvalidXLogRecPtr; + } else { + /* Parse the argument as LSN */ + opts->skiplsn = DatumGetLSN(DirectFunctionCall1(pg_lsn_in, CStringGetDatum(lsn_str))); + + if (XLogRecPtrIsInvalid(opts->skiplsn)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid WAL location (LSN): %s", lsn_str))); + } } else { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("unrecognized subscription parameter: %s", defel->defname))); @@ -185,38 +228,39 @@ static void parse_subscription_options(const List *options, char **conninfo, Lis * We've been explicitly asked to not connect, that requires some * additional processing. */ - if (connect && !*connect) { + if (IsSet(supported_opts, SUBOPT_CONNECT) && !opts->connect) { /* Check for incompatible options from the user. */ - if (*enabled_given && *enabled) + if (IsSet(opts->specified_opts, SUBOPT_ENABLED) && opts->enabled) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("%s and %s are mutually exclusive options", "connect = false", "enabled = true"))); - if (*copy_data_given && *copy_data) + if (IsSet(opts->specified_opts, SUBOPT_COPY_DATA) && opts->copy_data) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("%s and %s are mutually exclusive options", "connect = false", "copy_data = true"))); /* Change the defaults of other options. */ - *enabled = false; - *copy_data = false; + opts->enabled = false; + opts->copy_data = false; } /* * Do additional checking for disallowed combination when * slot_name = NONE was used. */ - if (slot_name && *slot_name_given && !*slot_name) { - if (enabled && *enabled_given && *enabled) { + if (!opts->slot_name && IsSet(supported_opts, SUBOPT_SLOT_NAME) && IsSet(opts->specified_opts, SUBOPT_SLOT_NAME)) { + if (opts->enabled && IsSet(supported_opts, SUBOPT_ENABLED) && IsSet(opts->specified_opts, SUBOPT_ENABLED)) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("slot_name = NONE and enabled = true are mutually exclusive options"))); } - if (enabled && !*enabled_given && *enabled) { + if (opts->enabled && IsSet(supported_opts, SUBOPT_ENABLED) && !IsSet(opts->specified_opts, SUBOPT_ENABLED)) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("subscription with slot_name = NONE must also set enabled = false"))); } } - if (copy_data && *copy_data && u_sess->attr.attr_storage.max_sync_workers_per_subscription == 0) { + if (IsSet(supported_opts, SUBOPT_COPY_DATA) && IsSet(opts->specified_opts, SUBOPT_COPY_DATA) && + u_sess->attr.attr_storage.max_sync_workers_per_subscription == 0) { ereport(WARNING, (errmsg("you need to set max_sync_workers_per_subscription because it is zero but " "copy_data is true"))); } @@ -451,27 +495,20 @@ ObjectAddress CreateSubscription(CreateSubscriptionStmt *stmt, bool isTopLevel) Datum values[Natts_pg_subscription]; Oid owner = GetUserId(); HeapTuple tup; - bool enabled_given = false; - bool enabled = true; - char *synchronous_commit; - char *slotname; - bool slotname_given; - bool binary; - bool binary_given; - bool copy_data; - bool copy_data_given; - bool connect; - bool connect_given; char originname[NAMEDATALEN]; List *publications; + bits32 supported_opts; + SubOpts opts = {0}; + opts.enabled = true; int rc; /* * Parse and check options. * Connection and publication should not be specified here. */ - parse_subscription_options(stmt->options, NULL, NULL, &enabled_given, &enabled, &slotname_given, &slotname, - &synchronous_commit, &binary_given, &binary, ©_data_given, ©_data, &connect_given, &connect); + supported_opts = (SUBOPT_ENABLED | SUBOPT_SLOT_NAME | SUBOPT_SYNCHRONOUS_COMMIT | SUBOPT_BINARY | + SUBOPT_COPY_DATA | SUBOPT_CONNECT); + parse_subscription_options(stmt->options, supported_opts, &opts); /* * Since creating a replication slot is not transactional, rolling back @@ -479,7 +516,7 @@ ObjectAddress CreateSubscription(CreateSubscriptionStmt *stmt, bool isTopLevel) * CREATE SUBSCRIPTION inside a transaction block if creating a * replication slot. */ - if (enabled) + if (opts.enabled) PreventTransactionChain(isTopLevel, "CREATE SUBSCRIPTION ... WITH (enabled = true)"); if (!superuser()) @@ -495,8 +532,8 @@ ObjectAddress CreateSubscription(CreateSubscriptionStmt *stmt, bool isTopLevel) } /* The default for synchronous_commit of subscriptions is off. */ - if (synchronous_commit == NULL) { - synchronous_commit = "off"; + if (opts.synchronous_commit == NULL) { + opts.synchronous_commit = "off"; } publications = stmt->publication; @@ -513,28 +550,29 @@ ObjectAddress CreateSubscription(CreateSubscriptionStmt *stmt, bool isTopLevel) values[Anum_pg_subscription_subdbid - 1] = ObjectIdGetDatum(u_sess->proc_cxt.MyDatabaseId); values[Anum_pg_subscription_subname - 1] = DirectFunctionCall1(namein, CStringGetDatum(stmt->subname)); values[Anum_pg_subscription_subowner - 1] = ObjectIdGetDatum(owner); - values[Anum_pg_subscription_subenabled - 1] = BoolGetDatum(enabled); - values[Anum_pg_subscription_subbinary - 1] = BoolGetDatum(binary); + values[Anum_pg_subscription_subenabled - 1] = BoolGetDatum(opts.enabled); + values[Anum_pg_subscription_subbinary - 1] = BoolGetDatum(opts.binary); /* encrypt conninfo */ char *encryptConninfo = EncryptOrDecryptConninfo(stmt->conninfo, 'E'); values[Anum_pg_subscription_subconninfo - 1] = CStringGetTextDatum(encryptConninfo); - if (enabled) { - if (!slotname_given) { - slotname = stmt->subname; + if (opts.enabled) { + if (!IsSet(opts.specified_opts, SUBOPT_SLOT_NAME)) { + opts.slot_name = stmt->subname; } - values[Anum_pg_subscription_subslotname - 1] = DirectFunctionCall1(namein, CStringGetDatum(slotname)); + values[Anum_pg_subscription_subslotname - 1] = DirectFunctionCall1(namein, CStringGetDatum(opts.slot_name)); } else { - if (slotname_given && slotname) { + if (IsSet(opts.specified_opts, SUBOPT_SLOT_NAME) && opts.slot_name) { ereport(WARNING, (errmsg("When enabled=false, it is dangerous to set slot_name. " "This will cause wal log accumulation on the publisher, " "so slot_name will be forcibly set to NULL."))); } nulls[Anum_pg_subscription_subslotname - 1] = true; } - values[Anum_pg_subscription_subsynccommit - 1] = CStringGetTextDatum(synchronous_commit); + values[Anum_pg_subscription_subsynccommit - 1] = CStringGetTextDatum(opts.synchronous_commit); values[Anum_pg_subscription_subpublications - 1] = publicationListToArray(publications); + values[Anum_pg_subscription_subskiplsn - 1] = LsnGetTextDatum(InvalidXLogRecPtr); tup = heap_form_tuple(RelationGetDescr(rel), values, nulls); @@ -553,8 +591,8 @@ ObjectAddress CreateSubscription(CreateSubscriptionStmt *stmt, bool isTopLevel) * Connect to remote side to execute requested commands and fetch table * info. */ - if (connect) { - if (!AttemptConnectPublisher(encryptConninfo, slotname, true)) { + if (opts.connect) { + if (!AttemptConnectPublisher(encryptConninfo, opts.slot_name, true)) { ereport(ERROR, (errcode(ERRCODE_CONNECTION_FAILURE), errmsg("Failed to connect to publisher."))); } @@ -567,8 +605,8 @@ ObjectAddress CreateSubscription(CreateSubscriptionStmt *stmt, bool isTopLevel) * If requested, create the replication slot on remote side for our * newly created subscription. */ - Assert(!enabled || slotname); - CreateSlotInPublisherAndInsertSubRel(slotname, subid, publications, ©_data, enabled); + Assert(!opts.enabled || opts.slot_name); + CreateSlotInPublisherAndInsertSubRel(opts.slot_name, subid, publications, &opts.copy_data, opts.enabled); (WalReceiverFuncTable[GET_FUNC_IDX]).walrcv_disconnect(); } else { @@ -582,7 +620,7 @@ ObjectAddress CreateSubscription(CreateSubscriptionStmt *stmt, bool isTopLevel) securec_check(rc, "", ""); /* Don't wake up logical replication launcher unnecessarily */ - if (enabled) { + if (opts.enabled) { ApplyLauncherWakeupAtCommit(); } @@ -793,17 +831,6 @@ ObjectAddress AlterSubscription(AlterSubscriptionStmt *stmt, bool isTopLevel) Datum values[Natts_pg_subscription]; HeapTuple tup; Oid subid; - bool enabled_given = false; - bool enabled = false; - bool binary_given = false; - bool binary = false; - char *synchronous_commit = NULL; - char *conninfo = NULL; - char *slot_name = NULL; - bool slotname_given = false; - bool copy_data = false; - bool copy_data_given = false; - List *publications = NIL; Subscription *sub = NULL; int rc; bool checkConn = false; @@ -812,6 +839,8 @@ ObjectAddress AlterSubscription(AlterSubscriptionStmt *stmt, bool isTopLevel) bool needFreeConninfo = false; char *finalSlotName = NULL; char *encryptConninfo = NULL; + bits32 supported_opts; + SubOpts opts = {0}; rel = heap_open(SubscriptionRelationId, RowExclusiveLock); @@ -831,17 +860,18 @@ ObjectAddress AlterSubscription(AlterSubscriptionStmt *stmt, bool isTopLevel) /* Lock the subscription so nobody else can do anything with it. */ LockSharedObject(SubscriptionRelationId, subid, 0, AccessExclusiveLock); - enabled = sub->enabled; + opts.enabled = sub->enabled; finalSlotName = sub->name; encryptConninfo = sub->conninfo; /* Parse options. */ if (!stmt->refresh) { - parse_subscription_options(stmt->options, &conninfo, &publications, &enabled_given, &enabled, &slotname_given, - &slot_name, &synchronous_commit, &binary_given, &binary, NULL, NULL, NULL, NULL); + supported_opts = (SUBOPT_CONNINFO | SUBOPT_PUBLICATION | SUBOPT_ENABLED | SUBOPT_SLOT_NAME | + SUBOPT_SYNCHRONOUS_COMMIT | SUBOPT_BINARY | SUBOPT_SKIPLSN); + parse_subscription_options(stmt->options, supported_opts, &opts); } else { - parse_subscription_options(stmt->options, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, - ©_data_given, ©_data, NULL, NULL); + supported_opts = SUBOPT_COPY_DATA; + parse_subscription_options(stmt->options, supported_opts, &opts); PreventTransactionChain(isTopLevel, "ALTER SUBSCRIPTION ... REFRESH"); } @@ -854,38 +884,39 @@ ObjectAddress AlterSubscription(AlterSubscriptionStmt *stmt, bool isTopLevel) rc = memset_s(replaces, sizeof(replaces), false, sizeof(replaces)); securec_check(rc, "", ""); - if (enabled_given && enabled != sub->enabled) { - values[Anum_pg_subscription_subenabled - 1] = BoolGetDatum(enabled); + if (IsSet(opts.specified_opts, SUBOPT_ENABLED) && opts.enabled != sub->enabled) { + values[Anum_pg_subscription_subenabled - 1] = BoolGetDatum(opts.enabled); replaces[Anum_pg_subscription_subenabled - 1] = true; } - if (conninfo) { + if (opts.conninfo) { /* Check the connection info string. */ - libpqrcv_check_conninfo(conninfo); - encryptConninfo = EncryptOrDecryptConninfo(conninfo, 'E'); - rc = memset_s(conninfo, strlen(conninfo), 0, strlen(conninfo)); + libpqrcv_check_conninfo(opts.conninfo); + encryptConninfo = EncryptOrDecryptConninfo(opts.conninfo, 'E'); + rc = memset_s(opts.conninfo, strlen(opts.conninfo), 0, strlen(opts.conninfo)); securec_check(rc, "\0", "\0"); values[Anum_pg_subscription_subconninfo - 1] = CStringGetTextDatum(encryptConninfo); replaces[Anum_pg_subscription_subconninfo - 1] = true; needFreeConninfo = true; /* need to check whether new conninfo can be used to connect to new publisher */ - if (sub->enabled || (enabled_given && enabled)) { + if (sub->enabled || (IsSet(opts.specified_opts, SUBOPT_ENABLED) && opts.enabled)) { checkConn = true; } } - if (slotname_given) { - if (sub->enabled && !slot_name) { + if (IsSet(opts.specified_opts, SUBOPT_SLOT_NAME)) { + if (sub->enabled && !opts.slot_name) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("cannot set slot_name = NONE for enabled subscription"))); } /* change to non-null value */ - if (slot_name) { - if (sub->enabled || (enabled_given && enabled)) { - values[Anum_pg_subscription_subslotname - 1] = DirectFunctionCall1(namein, CStringGetDatum(slot_name)); + if (opts.slot_name) { + if (sub->enabled || (IsSet(opts.specified_opts, SUBOPT_ENABLED) && opts.enabled)) { + values[Anum_pg_subscription_subslotname - 1] = DirectFunctionCall1(namein, + CStringGetDatum(opts.slot_name)); /* if old slotname is null or same as new slot name, then we need to validate the new slot name */ - validateSlot = sub->slotname == NULL || strcmp(slot_name, sub->slotname) != 0; - finalSlotName = slot_name; + validateSlot = sub->slotname == NULL || strcmp(opts.slot_name, sub->slotname) != 0; + finalSlotName = opts.slot_name; } else { ereport(ERROR, (errmsg("Currently enabled=false, cannot change slot_name to a non-null value."))); } @@ -896,30 +927,34 @@ ObjectAddress AlterSubscription(AlterSubscriptionStmt *stmt, bool isTopLevel) * when enable this subscription but slot_name is not specified, * it will be set to default value subname. */ - if (!sub->enabled && enabled_given && enabled) { + if (!sub->enabled && IsSet(opts.specified_opts, SUBOPT_ENABLED) && opts.enabled) { values[Anum_pg_subscription_subslotname - 1] = DirectFunctionCall1(namein, CStringGetDatum(sub->name)); } else { nulls[Anum_pg_subscription_subslotname - 1] = true; } } replaces[Anum_pg_subscription_subslotname - 1] = true; - } else if (!sub->enabled && enabled_given && enabled) { + } else if (!sub->enabled && IsSet(opts.specified_opts, SUBOPT_ENABLED) && opts.enabled) { values[Anum_pg_subscription_subslotname - 1] = DirectFunctionCall1(namein, CStringGetDatum(sub->name)); replaces[Anum_pg_subscription_subslotname - 1] = true; } - if (synchronous_commit) { - values[Anum_pg_subscription_subsynccommit - 1] = CStringGetTextDatum(synchronous_commit); + if (opts.synchronous_commit) { + values[Anum_pg_subscription_subsynccommit - 1] = CStringGetTextDatum(opts.synchronous_commit); replaces[Anum_pg_subscription_subsynccommit - 1] = true; } - if (binary_given) { - values[Anum_pg_subscription_subbinary - 1] = BoolGetDatum(binary); + if (IsSet(opts.specified_opts, SUBOPT_BINARY)) { + values[Anum_pg_subscription_subbinary - 1] = BoolGetDatum(opts.binary); replaces[Anum_pg_subscription_subbinary - 1] = true; } - if (publications != NIL) { - values[Anum_pg_subscription_subpublications - 1] = publicationListToArray(publications); + if (opts.publications != NIL) { + values[Anum_pg_subscription_subpublications - 1] = publicationListToArray(opts.publications); replaces[Anum_pg_subscription_subpublications - 1] = true; } else { - publications = sub->publications; + opts.publications = sub->publications; + } + if (IsSet(opts.specified_opts, SUBOPT_SKIPLSN)) { + values[Anum_pg_subscription_subskiplsn - 1] = LsnGetTextDatum(opts.skiplsn); + replaces[Anum_pg_subscription_subskiplsn - 1] = true; } tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls, replaces); @@ -938,8 +973,9 @@ ObjectAddress AlterSubscription(AlterSubscriptionStmt *stmt, bool isTopLevel) /* case 0: keep the subscription enabled, it's over here */ /* case 1: deactivating subscription */ - if (sub->enabled && !enabled) { - ereport(ERROR, (errmsg("If you want to deactivate this subscription, use DROP SUBSCRIPTION."))); + if (sub->enabled && !opts.enabled) { + ereport(WARNING, (errmsg("The subscription will be disabled, take care of the xlog would be accumulate " + "because the slot of subscription wouldn't be advanced"))); } /* case 2: keep the subscription active */ @@ -947,7 +983,7 @@ ObjectAddress AlterSubscription(AlterSubscriptionStmt *stmt, bool isTopLevel) if (!AttemptConnectPublisher(encryptConninfo, finalSlotName, true)) { ereport(ERROR, (errcode(ERRCODE_CONNECTION_FAILURE), errmsg( "Failed to connect to publisher."))); } - if (!CheckPublicationsExistOnPublisher(publications)) { + if (!CheckPublicationsExistOnPublisher(opts.publications)) { (WalReceiverFuncTable[GET_FUNC_IDX]).walrcv_disconnect(); ereport(ERROR, (errmsg("There are some publications not exist on the publisher."))); } @@ -958,7 +994,7 @@ ObjectAddress AlterSubscription(AlterSubscriptionStmt *stmt, bool isTopLevel) * enabling subscription, but slot hasn't been created, * then mark createSlot to true. */ - if (!sub->enabled && enabled && (!sub->slotname || !*(sub->slotname))) { + if (!sub->enabled && opts.enabled && (!sub->slotname || !*(sub->slotname))) { createSlot = true; } @@ -968,13 +1004,13 @@ ObjectAddress AlterSubscription(AlterSubscriptionStmt *stmt, bool isTopLevel) checkConn ? "The new conninfo cannot connect to new publisher." : "Failed to connect to publisher."))); } - if (!CheckPublicationsExistOnPublisher(publications)) { + if (!CheckPublicationsExistOnPublisher(opts.publications)) { (WalReceiverFuncTable[GET_FUNC_IDX]).walrcv_disconnect(); ereport(ERROR, (errmsg("There are some publications not exist on the publisher."))); } if (createSlot) { - CreateSlotInPublisherAndInsertSubRel(finalSlotName, subid, publications, NULL, true); + CreateSlotInPublisherAndInsertSubRel(finalSlotName, subid, opts.publications, NULL, true); } /* no need to validate replication slot if the slot is created just by ourself */ @@ -984,6 +1020,8 @@ ObjectAddress AlterSubscription(AlterSubscriptionStmt *stmt, bool isTopLevel) (WalReceiverFuncTable[GET_FUNC_IDX]).walrcv_disconnect(); ApplyLauncherWakeupAtCommit(); + } else if (!sub->enabled && opts.enabled) { + ApplyLauncherWakeupAtCommit(); } if (stmt->refresh) { @@ -991,7 +1029,7 @@ ObjectAddress AlterSubscription(AlterSubscriptionStmt *stmt, bool isTopLevel) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("ALTER SUBSCRIPTION ... REFRESH is not allowed for disabled subscriptions"))); } - AlterSubscription_refresh(sub, copy_data); + AlterSubscription_refresh(sub, opts.copy_data); } if (needFreeConninfo) { diff --git a/src/gausskernel/runtime/executor/execReplication.cpp b/src/gausskernel/runtime/executor/execReplication.cpp index d0e627e15..91802d685 100644 --- a/src/gausskernel/runtime/executor/execReplication.cpp +++ b/src/gausskernel/runtime/executor/execReplication.cpp @@ -53,7 +53,8 @@ static bool RelationFindReplTupleSeq(Relation rel, LockTupleMode lockmode, Tuple * This is not generic routine, it expects the idxrel to be replication * identity of a rel and meet all limitations associated with that. */ -static bool build_replindex_scan_key(ScanKey skey, Relation rel, Relation idxrel, TupleTableSlot *searchslot) +static bool build_replindex_scan_key(ScanKey skey, Relation rel, Relation idxrel, TupleTableSlot *searchslot, + EState *estate) { int attoff; bool isnull; @@ -66,6 +67,16 @@ static bool build_replindex_scan_key(ScanKey skey, Relation rel, Relation idxrel Assert(!isnull); opclass = (oidvector *)DatumGetPointer(indclassDatum); + List* expressionsState = NIL; + ExprContext* econtext = GetPerTupleExprContext(estate); + econtext->ecxt_scantuple = searchslot; + ListCell* indexpr_item = NULL; + + if (idxrel->rd_indexprs != NIL) { + expressionsState = ExecPrepareExprList(idxrel->rd_indexprs, estate); + indexpr_item = list_head(expressionsState); + } + /* Build scankey for every attribute in the index. */ for (attoff = 0; attoff < IndexRelationGetNumberOfKeyAttributes(idxrel); attoff++) { Oid op; @@ -92,14 +103,34 @@ static bool build_replindex_scan_key(ScanKey skey, Relation rel, Relation idxrel regop = get_opcode(op); - /* Initialize the scankey. */ - ScanKeyInit(&skey[attoff], pkattno, BTEqualStrategyNumber, regop, searchslot->tts_values[mainattno - 1]); - skey[attoff].sk_collation = idxrel->rd_indcollation[attoff]; + if (mainattno != 0) { + /* Initialize the scankey. */ + ScanKeyInit(&skey[attoff], pkattno, BTEqualStrategyNumber, regop, searchslot->tts_values[mainattno - 1]); + skey[attoff].sk_collation = idxrel->rd_indcollation[attoff]; - /* Check for null value. */ - if (searchslot->tts_isnull[mainattno - 1]) { - hasnulls = true; - skey[attoff].sk_flags |= SK_ISNULL; + /* Check for null value. */ + if (searchslot->tts_isnull[mainattno - 1]) { + hasnulls = true; + skey[attoff].sk_flags |= SK_ISNULL; + } + } else { + if (idxrel->rd_indexprs == NIL) { + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("wrong number of index expressions"))); + } else { + bool isNull = false; + Datum datum = ExecEvalExprSwitchContext((ExprState*)lfirst(indexpr_item), econtext, &isNull, NULL); + indexpr_item = lnext(indexpr_item); + + ScanKeyInit(&skey[attoff], pkattno, BTEqualStrategyNumber, regop, datum); + skey[attoff].sk_collation = idxrel->rd_indcollation[attoff]; + + /* Check for null value. */ + if (isNull) { + hasnulls = true; + skey[attoff].sk_flags |= SK_ISNULL; + } + } } } @@ -307,7 +338,7 @@ static bool RelationFindReplTupleByIndex(EState *estate, Relation rel, Relation scan->isUpsert = true; /* Build scan key. */ - build_replindex_scan_key(skey, targetRel, idxrel, searchslot); + build_replindex_scan_key(skey, targetRel, idxrel, searchslot, estate); while (true) { found = false; diff --git a/src/gausskernel/storage/replication/logical/parallel_decode.cpp b/src/gausskernel/storage/replication/logical/parallel_decode.cpp index 2246bafbe..f04182377 100644 --- a/src/gausskernel/storage/replication/logical/parallel_decode.cpp +++ b/src/gausskernel/storage/replication/logical/parallel_decode.cpp @@ -72,13 +72,23 @@ ParallelReorderBufferTXN *ParallelReorderBufferGetOldestTXN(ParallelReorderBuffe return txn; } -void tuple_to_stringinfo(Relation relation, StringInfo s, TupleDesc tupdesc, HeapTuple tuple, bool isOld) +void tuple_to_stringinfo(Relation relation, StringInfo s, TupleDesc tupdesc, HeapTuple tuple, bool isOld, + bool printOid) { if ((tuple->tupTableType == HEAP_TUPLE) && (HEAP_TUPLE_IS_COMPRESSED(tuple->t_data) || (int)HeapTupleHeaderGetNatts(tuple->t_data, tupdesc) > tupdesc->natts)) { return; } + if (printOid) { + Oid oid; + + /* print oid of tuple, it's not included in the TupleDesc */ + if ((oid = HeapTupleHeaderGetOid(tuple->t_data)) != InvalidOid) { + appendStringInfo(s, " oid[oid]:%u", oid); + } + } + /* print all columns individually */ for (int natt = 0; natt < tupdesc->natts; natt++) { Form_pg_attribute attr; /* the attribute itself */ @@ -90,6 +100,12 @@ void tuple_to_stringinfo(Relation relation, StringInfo s, TupleDesc tupdesc, Hea attr = &tupdesc->attrs[natt]; + /* + * Don't print dropped columns, we can't be sure everything is + * available for them. + * Don't print system columns, oid will already have been printed if + * present. + */ if (attr->attisdropped || attr->attnum < 0 || (isOld && !IsRelationReplidentKey(relation, attr->attnum))) continue; diff --git a/src/gausskernel/storage/replication/logical/worker.cpp b/src/gausskernel/storage/replication/logical/worker.cpp index 1cc9aafb2..167333633 100644 --- a/src/gausskernel/storage/replication/logical/worker.cpp +++ b/src/gausskernel/storage/replication/logical/worker.cpp @@ -115,6 +115,10 @@ static void ApplyWorkerProcessMsg(char type, StringInfo s, XLogRecPtr *lastRcv); static void apply_dispatch(StringInfo s); static void apply_handle_conninfo(StringInfo s); static void UpdateConninfo(char* standbysInfo); +static Oid find_conflict_tuple(EState *estate, TupleTableSlot *remoteslot, TupleTableSlot *localslot, + FakeRelationPartition *fakeRelInfo, TupleTableSlot *originslot = NULL); +static void IsSkippingChanges(XLogRecPtr finish_lsn); +static void StopSkippingChanges(); /* * Should this worker apply changes for given relation. @@ -478,6 +482,8 @@ static void apply_handle_begin(StringInfo s) t_thrd.applyworker_cxt.remoteFinalLsn = begin_data.final_lsn; t_thrd.applyworker_cxt.curRemoteCsn = begin_data.csn; + IsSkippingChanges(begin_data.final_lsn); + pgstat_report_activity(STATE_RUNNING, NULL); } @@ -510,6 +516,10 @@ static void apply_handle_commit(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(commit_data.end_lsn); + if (t_thrd.applyworker_cxt.isSkipTransaction) { + StopSkippingChanges(); + } + pgstat_report_activity(STATE_IDLE, NULL); } @@ -570,6 +580,69 @@ static Oid GetRelationIdentityOrPK(Relation rel) return idxoid; } +/* + * Find the tuple in a table using any unique index and returns the conflicting + * index's oid, if any conflict found. + * + * *originslot* contains the old tuple during UPDATE, if conflict with it which + * to be updated, ignore it. + */ +Oid find_conflict_tuple(EState *estate, TupleTableSlot *remoteslot, TupleTableSlot *localslot, + FakeRelationPartition *fakeRelInfo, TupleTableSlot *originslot) +{ + Oid replidxoid = InvalidOid; + bool found = false; + ResultRelInfo* relinfo = estate->es_result_relation_info; + + /* Check the replica identity index first */ + replidxoid = RelationGetReplicaIndex(relinfo->ri_RelationDesc); + found = RelationFindReplTuple(estate, relinfo->ri_RelationDesc, replidxoid, LockTupleExclusive, remoteslot, + localslot, fakeRelInfo); + if (found) { + if (originslot != NULL && ItemPointerCompare(tableam_tops_get_t_self(relinfo->ri_RelationDesc, + localslot->tts_tuple), tableam_tops_get_t_self(relinfo->ri_RelationDesc, + originslot->tts_tuple)) == 0) { + /* If conflict with the tuple to be updated, ignore it. */ + found = false; + } else { + return replidxoid; + } + } + + for (int i = 0; i < relinfo->ri_NumIndices; i++) { + IndexInfo *ii = relinfo->ri_IndexRelationInfo[i]; + Relation idxrel; + Oid idxoid = InvalidOid; + + if (!ii->ii_Unique) { + continue; + } + + idxrel = relinfo->ri_IndexRelationDescs[i]; + idxoid = RelationGetRelid(idxrel); + + if (idxoid == replidxoid) { + continue; + } + + found = RelationFindReplTuple(estate, relinfo->ri_RelationDesc, idxoid, LockTupleExclusive, remoteslot, + localslot, fakeRelInfo); + + if (found) { + if (originslot != NULL && ItemPointerCompare(tableam_tops_get_t_self(relinfo->ri_RelationDesc, + localslot->tts_tuple), tableam_tops_get_t_self(relinfo->ri_RelationDesc, + originslot->tts_tuple)) == 0) { + /* If conflict with the tuple to be updated, ignore it. */ + found = false; + } else { + return idxoid; + } + } + } + + return InvalidOid; +} + /* * Handle INSERT message. */ @@ -582,6 +655,13 @@ static void apply_handle_insert(StringInfo s) TupleTableSlot *remoteslot; MemoryContext oldctx; FakeRelationPartition fakeRelInfo; + TupleTableSlot *localslot; + EPQState epqstate; + Oid conflictIndexOid = InvalidOid; + + if (t_thrd.applyworker_cxt.isSkipTransaction) { + return; + } ensure_transaction(); @@ -600,6 +680,8 @@ static void apply_handle_insert(StringInfo s) estate = create_estate_for_relation(rel); remoteslot = ExecInitExtraTupleSlot(estate, rel->localrel->rd_tam_ops); ExecSetSlotDescriptor(remoteslot, RelationGetDescr(rel->localrel)); + localslot = ExecInitExtraTupleSlot(estate, rel->localrel->rd_tam_ops); + ExecSetSlotDescriptor(localslot, RelationGetDescr(rel->localrel)); /* Input functions may need an active snapshot, so get one */ PushActiveSnapshot(GetTransactionSnapshot()); @@ -611,11 +693,62 @@ static void apply_handle_insert(StringInfo s) ExecOpenIndices(estate->es_result_relation_info, false); - /* Get fake relation and partition for patitioned table */ - GetFakeRelAndPart(estate, rel->localrel, remoteslot, &fakeRelInfo); + if ((conflictIndexOid = find_conflict_tuple(estate, remoteslot, localslot, &fakeRelInfo)) != InvalidOid) { + StringInfoData localtup, remotetup; + initStringInfo(&localtup); + tuple_to_stringinfo(rel->localrel, &localtup, RelationGetDescr(rel->localrel), + (HeapTuple)localslot->tts_tuple, false); + + initStringInfo(&remotetup); + tuple_to_stringinfo(rel->localrel, &remotetup, RelationGetDescr(rel->localrel), + (HeapTuple)tableam_tslot_get_tuple_from_slot(rel->localrel, remoteslot), false); + + switch (u_sess->attr.attr_storage.subscription_conflict_resolution) { + case RESOLVE_ERROR: + ereport(ERROR, (errmsg("CONFLICT: remote insert on relation %s (local index %s). Resolution: error.", + RelationGetRelationName(rel->localrel), get_rel_name(conflictIndexOid)), + errdetail("local tuple: %s, remote tuple: %s, origin: pg_%u, commit_lsn: %X/%X", localtup.data, + remotetup.data, t_thrd.applyworker_cxt.curWorker->subid, + (uint32)(t_thrd.applyworker_cxt.remoteFinalLsn >> BITS_PER_INT), + (uint32)t_thrd.applyworker_cxt.remoteFinalLsn))); + break; + case RESOLVE_APPLY_REMOTE: + ereport(LOG, (errmsg("CONFLICT: remote insert on relation %s (local index %s). " + "Resolution: apply_remote.", + RelationGetRelationName(rel->localrel), get_rel_name(conflictIndexOid)), + errdetail("local tuple: %s, remote tuple: %s, origin: pg_%u, commit_lsn: %X/%X", localtup.data, + remotetup.data, t_thrd.applyworker_cxt.curWorker->subid, + (uint32)(t_thrd.applyworker_cxt.remoteFinalLsn >> BITS_PER_INT), + (uint32)t_thrd.applyworker_cxt.remoteFinalLsn))); + EvalPlanQualInit(&epqstate, estate, NULL, NIL, -1); + EvalPlanQualSetSlot(&epqstate, remoteslot); + /* Do the actual update. */ + ExecSimpleRelationUpdate(estate, &epqstate, localslot, remoteslot, &fakeRelInfo); + + EvalPlanQualEnd(&epqstate); + break; + case RESOLVE_KEEP_LOCAL: + ereport(LOG, (errmsg("CONFLICT: remote insert on relation %s (local index %s). " + "Resolution: keep_local.", + RelationGetRelationName(rel->localrel), get_rel_name(conflictIndexOid)), + errdetail("local tuple: %s, remote tuple: %s, origin: pg_%u, commit_lsn: %X/%X", localtup.data, + remotetup.data, t_thrd.applyworker_cxt.curWorker->subid, + (uint32)(t_thrd.applyworker_cxt.remoteFinalLsn >> BITS_PER_INT), + (uint32)t_thrd.applyworker_cxt.remoteFinalLsn))); + break; + default: + ereport(ERROR, (errmsg("wrong parameter value for subscription_conflict_resolution"))); + break; + } + FreeStringInfo(&localtup); + FreeStringInfo(&remotetup); + } else { + /* Get fake relation and partition for patitioned table */ + GetFakeRelAndPart(estate, rel->localrel, remoteslot, &fakeRelInfo); - /* Do the insert. */ - ExecSimpleRelationInsert(estate, remoteslot, &fakeRelInfo); + /* Do the insert. */ + ExecSimpleRelationInsert(estate, remoteslot, &fakeRelInfo); + } /* Cleanup. */ ExecCloseIndices(estate->es_result_relation_info); @@ -701,10 +834,16 @@ static void apply_handle_update(StringInfo s) bool has_oldtup; TupleTableSlot *localslot; TupleTableSlot *remoteslot; + TupleTableSlot *conflictLocalSlot; RangeTblEntry *target_rte = NULL; bool found = false; MemoryContext oldctx; FakeRelationPartition fakeRelInfo; + Oid conflictIndexOid = InvalidOid; + + if (t_thrd.applyworker_cxt.isSkipTransaction) { + return; + } ensure_transaction(); @@ -728,6 +867,8 @@ static void apply_handle_update(StringInfo s) ExecSetSlotDescriptor(remoteslot, RelationGetDescr(rel->localrel)); localslot = ExecInitExtraTupleSlot(estate, rel->localrel->rd_tam_ops); ExecSetSlotDescriptor(localslot, RelationGetDescr(rel->localrel)); + conflictLocalSlot = ExecInitExtraTupleSlot(estate, rel->localrel->rd_tam_ops); + ExecSetSlotDescriptor(conflictLocalSlot, RelationGetDescr(rel->localrel)); EvalPlanQualInit(&epqstate, estate, NULL, NIL, -1); /* @@ -783,10 +924,64 @@ static void apply_handle_update(StringInfo s) slot_modify_data(remoteslot, localslot, rel, &newtup); MemoryContextSwitchTo(oldctx); - EvalPlanQualSetSlot(&epqstate, remoteslot); + if ((conflictIndexOid = find_conflict_tuple(estate, remoteslot, conflictLocalSlot, &fakeRelInfo, localslot)) + != InvalidOid) { + StringInfoData localtup, remotetup; + initStringInfo(&localtup); + tuple_to_stringinfo(rel->localrel, &localtup, RelationGetDescr(rel->localrel), + (HeapTuple)conflictLocalSlot->tts_tuple, false); + + initStringInfo(&remotetup); + tuple_to_stringinfo(rel->localrel, &remotetup, RelationGetDescr(rel->localrel), + (HeapTuple)tableam_tslot_get_tuple_from_slot(rel->localrel, remoteslot), false); + + switch (u_sess->attr.attr_storage.subscription_conflict_resolution) { + case RESOLVE_ERROR: + ereport(ERROR, (errmsg("CONFLICT: remote update on relation %s (local index %s). " + "Resolution: error.", + RelationGetRelationName(rel->localrel), get_rel_name(conflictIndexOid)), + errdetail("local tuple: %s, remote tuple: %s, origin: pg_%u, commit_lsn: %X/%X", localtup.data, + remotetup.data, t_thrd.applyworker_cxt.curWorker->subid, + (uint32)(t_thrd.applyworker_cxt.remoteFinalLsn >> BITS_PER_INT), + (uint32)t_thrd.applyworker_cxt.remoteFinalLsn))); + break; + case RESOLVE_APPLY_REMOTE: + ereport(LOG, (errmsg("CONFLICT: remote update on relation %s (local index %s). " + "Resolution: apply_remote.", + RelationGetRelationName(rel->localrel), get_rel_name(conflictIndexOid)), + errdetail("local tuple: %s, remote tuple: %s, origin: pg_%u, commit_lsn: %X/%X", localtup.data, + remotetup.data, t_thrd.applyworker_cxt.curWorker->subid, + (uint32)(t_thrd.applyworker_cxt.remoteFinalLsn >> BITS_PER_INT), + (uint32)t_thrd.applyworker_cxt.remoteFinalLsn))); + /* first delete the conflict tuple */ + EvalPlanQualSetSlot(&epqstate, conflictLocalSlot); + ExecSimpleRelationDelete(estate, &epqstate, conflictLocalSlot, &fakeRelInfo); + + EvalPlanQualSetSlot(&epqstate, remoteslot); + /* Do the actual update. */ + ExecSimpleRelationUpdate(estate, &epqstate, localslot, remoteslot, &fakeRelInfo); + break; + case RESOLVE_KEEP_LOCAL: + ereport(LOG, (errmsg("CONFLICT: remote update on relation %s (local index %s). " + "Resolution: keep_local.", + RelationGetRelationName(rel->localrel), get_rel_name(conflictIndexOid)), + errdetail("local tuple: %s, remote tuple: %s, origin: pg_%u, commit_lsn: %X/%X", localtup.data, + remotetup.data, t_thrd.applyworker_cxt.curWorker->subid, + (uint32)(t_thrd.applyworker_cxt.remoteFinalLsn >> BITS_PER_INT), + (uint32)t_thrd.applyworker_cxt.remoteFinalLsn))); + break; + default: + ereport(ERROR, (errmsg("wrong parameter value for subscription_conflict_resolution"))); + break; + } + FreeStringInfo(&localtup); + FreeStringInfo(&remotetup); + } else { + EvalPlanQualSetSlot(&epqstate, remoteslot); - /* Do the actual update. */ - ExecSimpleRelationUpdate(estate, &epqstate, localslot, remoteslot, &fakeRelInfo); + /* Do the actual update. */ + ExecSimpleRelationUpdate(estate, &epqstate, localslot, remoteslot, &fakeRelInfo); + } } else { /* * The tuple to be updated could not be found. @@ -824,6 +1019,10 @@ static void apply_handle_delete(StringInfo s) MemoryContext oldctx; FakeRelationPartition fakeRelInfo; + if (t_thrd.applyworker_cxt.isSkipTransaction) { + return; + } + ensure_transaction(); relid = logicalrep_read_delete(s, &oldtup); @@ -1828,3 +2027,78 @@ bool IsLogicalWorker(void) { return t_thrd.applyworker_cxt.curWorker != NULL; } + +/* + * Start skipping changes of the transaction if the given LSN matches the + * LSN specified by subscription's skiplsn. + */ +static void IsSkippingChanges(XLogRecPtr finish_lsn) +{ + /* + * Quick return if it's not requested to skip this transaction. This + * function is called for every remote transaction and we assume that + * skipping the transaction is not used often. + */ + if (likely(XLogRecPtrIsInvalid(t_thrd.applyworker_cxt.mySubscription->skiplsn) || + t_thrd.applyworker_cxt.mySubscription->skiplsn != finish_lsn)) { + return; + } + + t_thrd.applyworker_cxt.isSkipTransaction = true; +} + +static void StopSkippingChanges() +{ + t_thrd.applyworker_cxt.isSkipTransaction = false; + + /* + * Quick return if it's not requested to skip this transaction. This + * function is called for every remote transaction and we assume that + * skipping the transaction is not used often. + */ + if (!IsTransactionState()) { + StartTransactionCommand(); + } + + HeapTuple tup; + Relation rel; + bool nulls[Natts_pg_subscription]; + bool replaces[Natts_pg_subscription]; + Datum values[Natts_pg_subscription]; + errno_t rc = 0; + + /* + * Protect subskiplsn of pg_subscription from being concurrently updated + * while clearing it. + */ + LockSharedObject(SubscriptionRelationId, t_thrd.applyworker_cxt.mySubscription->oid, 0, AccessShareLock); + rel = heap_open(SubscriptionRelationId, RowExclusiveLock); + + /* Fetch the existing tuple. */ + tup = SearchSysCacheCopy1(SUBSCRIPTIONOID, ObjectIdGetDatum(t_thrd.applyworker_cxt.mySubscription->oid)); + + if (!HeapTupleIsValid(tup)) { + ereport(ERROR, (errmsg("subscription \"%s\" does not exist", t_thrd.applyworker_cxt.mySubscription->name))); + } + + rc = memset_s(values, sizeof(values),0, sizeof(values)); + securec_check_c(rc, "\0", "\0"); + rc = memset_s(nulls, sizeof(nulls),false, sizeof(nulls)); + securec_check_c(rc, "\0", "\0"); + rc = memset_s(replaces, sizeof(replaces), false, sizeof(replaces)); + securec_check_c(rc, "\0", "\0"); + + /* reset subskiplsn */ + values[Anum_pg_subscription_subskiplsn - 1] = LsnGetTextDatum(InvalidXLogRecPtr); + replaces[Anum_pg_subscription_subskiplsn - 1] = true; + + tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls, replaces); + /* Update the catalog. */ + simple_heap_update(rel, &tup->t_self, tup); + CatalogUpdateIndexes(rel, tup); + + heap_freetuple(tup); + heap_close(rel, NoLock); + + CommitTransactionCommand(); +} diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h index 6765cd38c..8b79927fd 100644 --- a/src/include/catalog/pg_subscription.h +++ b/src/include/catalog/pg_subscription.h @@ -52,13 +52,15 @@ CATALOG(pg_subscription,6126) BKI_SHARED_RELATION BKI_ROWTYPE_OID(6128) BKI_SCHE text subpublications[1]; /* List of publications subscribed to */ bool subbinary; /* True if the subscription wants the * publisher to send data in binary */ + text subskiplsn; /* All changes finished at this LSN are + * skipped */ #endif } FormData_pg_subscription; typedef FormData_pg_subscription *Form_pg_subscription; -#define Natts_pg_subscription 9 +#define Natts_pg_subscription 10 #define Anum_pg_subscription_subdbid 1 #define Anum_pg_subscription_subname 2 #define Anum_pg_subscription_subowner 3 @@ -68,6 +70,7 @@ typedef FormData_pg_subscription *Form_pg_subscription; #define Anum_pg_subscription_subsynccommit 7 #define Anum_pg_subscription_subpublications 8 #define Anum_pg_subscription_subbinary 9 +#define Anum_pg_subscription_subskiplsn 10 typedef struct Subscription { @@ -81,6 +84,8 @@ typedef struct Subscription { char *synccommit; /* Synchronous commit setting for worker */ List *publications; /* List of publication names to subscribe to */ bool binary; /* Indicates if the subscription wants data in binary format */ + XLogRecPtr skiplsn; /* All changes finished at this LSN are + * skipped */ } Subscription; @@ -91,6 +96,8 @@ extern char *get_subscription_name(Oid subid, bool missing_ok); extern int CountDBSubscriptions(Oid dbid); extern void ClearListContent(List *list); +extern Datum LsnGetTextDatum(XLogRecPtr lsn); +extern XLogRecPtr TextDatumGetLsn(Datum datum); #endif /* PG_SUBSCRIPTION_H */ diff --git a/src/include/knl/knl_guc/knl_session_attr_storage.h b/src/include/knl/knl_guc/knl_session_attr_storage.h index 96d0d72ff..af7a7e430 100755 --- a/src/include/knl/knl_guc/knl_session_attr_storage.h +++ b/src/include/knl/knl_guc/knl_session_attr_storage.h @@ -266,6 +266,7 @@ typedef struct knl_session_attr_storage { int logical_sender_timeout; int ignore_standby_lsn_window; int ignore_feedback_xmin_window; + int subscription_conflict_resolution; } knl_session_attr_storage; #endif /* SRC_INCLUDE_KNL_KNL_SESSION_ATTR_STORAGE */ diff --git a/src/include/knl/knl_thread.h b/src/include/knl/knl_thread.h index 06cd5f6ce..fee147d2b 100755 --- a/src/include/knl/knl_thread.h +++ b/src/include/knl/knl_thread.h @@ -3335,6 +3335,7 @@ typedef struct knl_t_apply_worker_context { List *tableStates; XLogRecPtr remoteFinalLsn; CommitSeqNo curRemoteCsn; + bool isSkipTransaction; } knl_t_apply_worker_context; typedef struct knl_t_publication_context { diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h index b8baace4b..9b9e44e00 100644 --- a/src/include/replication/logical.h +++ b/src/include/replication/logical.h @@ -358,5 +358,7 @@ extern void FreeLogicalLog(ParallelReorderBuffer *rb, logicalLog *logChange, int extern bool LogicalDecodeParseOptionsDefault(const char* defaultStr, void **options); extern DecodeOptionsDefault* LogicalDecodeGetOptionsDefault(); template void LogicalDecodeReportLostChanges(const T *iterstate); +extern void tuple_to_stringinfo(Relation relation, StringInfo s, TupleDesc tupdesc, HeapTuple tuple, bool isOld, + bool printOid = false); #endif diff --git a/src/include/replication/replicainternal.h b/src/include/replication/replicainternal.h index b582b44e6..924601b2b 100755 --- a/src/include/replication/replicainternal.h +++ b/src/include/replication/replicainternal.h @@ -214,6 +214,13 @@ typedef enum replauthmode{ REPL_AUTH_UUID /* uuid auth */ } ReplAuthMode; +typedef enum +{ + RESOLVE_ERROR, + RESOLVE_APPLY_REMOTE, + RESOLVE_KEEP_LOCAL +} PGLogicalResolveOption; + extern bool data_catchup; extern bool wal_catchup; extern BuildMode build_mode; diff --git a/src/test/regress/input/subscription.source b/src/test/regress/input/subscription.source index 30cd7a96b..3083dc5b6 100644 --- a/src/test/regress/input/subscription.source +++ b/src/test/regress/input/subscription.source @@ -70,6 +70,14 @@ ALTER SUBSCRIPTION testsub owner to regress_subscription_user2; -- alter subbinary to true ALTER SUBSCRIPTION testsub SET (binary=true); select subname, subbinary from pg_subscription where subname='testsub'; +-- set subskiplsn +ALTER SUBSCRIPTION testsub SET (skiplsn = '0/ABCDEF'); +select subname, subskiplsn from pg_subscription where subname='testsub'; +ALTER SUBSCRIPTION testsub SET (skiplsn = '0/ABCDEFGH'); +ALTER SUBSCRIPTION testsub SET (skiplsn = 'none'); +select subname, subskiplsn from pg_subscription where subname='testsub'; +-- disable test +ALTER SUBSCRIPTION testsub DISABLE; --rename ALTER SUBSCRIPTION testsub rename to testsub_rename; --- inside a transaction block diff --git a/src/test/regress/output/recovery_2pc_tools.source b/src/test/regress/output/recovery_2pc_tools.source index fc237cfbd..55bbb3763 100644 --- a/src/test/regress/output/recovery_2pc_tools.source +++ b/src/test/regress/output/recovery_2pc_tools.source @@ -662,6 +662,7 @@ select name,vartype,unit,min_val,max_val from pg_settings where name <> 'qunit_c stats_temp_directory | string | | | stream_cluster_run_mode | enum | | | string_hash_compatible | bool | | | + subscription_conflict_resolution | enum | | | support_batch_bind | bool | | | support_extended_features | bool | | | sync_config_strategy | enum | | | diff --git a/src/test/regress/output/subscription.source b/src/test/regress/output/subscription.source index 7ef7842a1..24fe8ecaa 100644 --- a/src/test/regress/output/subscription.source +++ b/src/test/regress/output/subscription.source @@ -19,6 +19,7 @@ ALTER SUBSCRIPTION name CONNECTION 'conninfo' ALTER SUBSCRIPTION name SET PUBLICATION publication_name [, ...] ALTER SUBSCRIPTION name REFRESH PUBLICATION [ WITH ( refresh_option [= value] [, ... ] ) ] ALTER SUBSCRIPTION name ENABLE +ALTER SUBSCRIPTION name DISABLE ALTER SUBSCRIPTION name SET ( subscription_parameter [= value] [, ... ] ) ALTER SUBSCRIPTION name OWNER TO new_owner ALTER SUBSCRIPTION name RENAME TO new_name @@ -153,6 +154,25 @@ select subname, subbinary from pg_subscription where subname='testsub'; testsub | t (1 row) +-- set subskiplsn +ALTER SUBSCRIPTION testsub SET (skiplsn = '0/ABCDEF'); +select subname, subskiplsn from pg_subscription where subname='testsub'; + subname | subskiplsn +---------+------------ + testsub | 0/ABCDEF +(1 row) + +ALTER SUBSCRIPTION testsub SET (skiplsn = '0/ABCDEFGH'); +ERROR: invalid input syntax for type pg_lsn: "0/ABCDEFGH" +ALTER SUBSCRIPTION testsub SET (skiplsn = 'none'); +select subname, subskiplsn from pg_subscription where subname='testsub'; + subname | subskiplsn +---------+------------ + testsub | 0/0 +(1 row) + +-- disable test +ALTER SUBSCRIPTION testsub DISABLE; --rename ALTER SUBSCRIPTION testsub rename to testsub_rename; --- inside a transaction block @@ -337,12 +357,15 @@ SELECT object_name,detail_info FROM pg_query_audit('2022-01-13 9:30:00', '2031-1 testsub | ALTER SUBSCRIPTION testsub SET (synchronous_commit=on); testsub | ALTER SUBSCRIPTION testsub owner to regress_subscription_user2; testsub | ALTER SUBSCRIPTION testsub SET (binary=true); + testsub | ALTER SUBSCRIPTION testsub SET (skiplsn = '0/ABCDEF'); + testsub | ALTER SUBSCRIPTION testsub SET (skiplsn = 'none'); + testsub | ALTER SUBSCRIPTION testsub DISABLE; testsub | ALTER SUBSCRIPTION testsub rename to testsub_rename; sub_len_999 | CREATE SUBSCRIPTION sub_len_999 CONNECTION *********************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************PUBLICATION insert_only WITH (connect = false); testsub_rename | DROP SUBSCRIPTION IF EXISTS testsub_rename; testsub_maskconninfo | DROP SUBSCRIPTION IF EXISTS testsub_maskconninfo; sub_len_999 | DROP SUBSCRIPTION IF EXISTS sub_len_999; -(17 rows) +(20 rows) --clear audit log SELECT pg_delete_audit('1012-11-10', '3012-11-11'); diff --git a/src/test/subscription/schedule b/src/test/subscription/schedule index 6913c6c8d..52ee04318 100644 --- a/src/test/subscription/schedule +++ b/src/test/subscription/schedule @@ -10,4 +10,6 @@ sync encoding ddl matviews -change_wal_level \ No newline at end of file +change_wal_level +skiplsn +disable \ No newline at end of file diff --git a/src/test/subscription/testcase/disable.sh b/src/test/subscription/testcase/disable.sh new file mode 100644 index 000000000..179769002 --- /dev/null +++ b/src/test/subscription/testcase/disable.sh @@ -0,0 +1,65 @@ +#!/bin/sh + +source $1/env_utils.sh $1 $2 + +case_db="disable_db" + +function test_1() { + echo "create database and tables." + exec_sql $db $pub_node1_port "CREATE DATABASE $case_db" + exec_sql $db $sub_node1_port "CREATE DATABASE $case_db" + # Create some preexisting content on publisher + exec_sql $case_db $pub_node1_port "CREATE TABLE tab_rep (a int primary key, b int)" + + # Setup structure on subscriber + exec_sql $case_db $sub_node1_port "CREATE TABLE tab_rep (a int primary key, b int)" + + # Setup logical replication + echo "create publication and subscription." + publisher_connstr="port=$pub_node1_port host=$g_local_ip dbname=$case_db user=$username password=$passwd" + exec_sql $case_db $pub_node1_port "CREATE PUBLICATION tap_pub FOR ALL TABLES" + exec_sql $case_db $sub_node1_port "CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr' PUBLICATION tap_pub" + + # Wait for initial table sync to finish + wait_for_subscription_sync $case_db $sub_node1_port + + exec_sql $case_db $pub_node1_port "insert into tab_rep values (1,1)" + + wait_for_catchup $case_db $pub_node1_port "tap_sub" + + exec_sql $case_db $sub_node1_port "alter subscription tap_sub disable" + + exec_sql $case_db $pub_node1_port "insert into tab_rep values (2,2)" + + if [ "$(exec_sql $case_db $sub_node1_port "SELECT * FROM tab_rep")" = "1|1" ]; then + echo "check data not sync after disable subscription success" + else + echo "$failed_keyword when check data not sync after disable subscription" + exit 1 + fi + + exec_sql $case_db $sub_node1_port "alter subscription tap_sub enable" + + wait_for_catchup $case_db $pub_node1_port "tap_sub" + + if [ "$(exec_sql $case_db $sub_node1_port "SELECT * FROM tab_rep")" = "1|1 +2|2" ]; then + echo "check data not sync after enable subscription success" + else + echo "$failed_keyword when check data not sync after enable subscription" + exit 1 + fi +} + +function tear_down() { + exec_sql $case_db $sub_node1_port "DROP SUBSCRIPTION IF EXISTS tap_sub" + exec_sql $case_db $pub_node1_port "DROP PUBLICATION IF EXISTS tap_pub" + + exec_sql $db $sub_node1_port "DROP DATABASE $case_db" + exec_sql $db $pub_node1_port "DROP DATABASE $case_db" + + echo "tear down" +} + +test_1 +tear_down \ No newline at end of file diff --git a/src/test/subscription/testcase/skiplsn.sh b/src/test/subscription/testcase/skiplsn.sh new file mode 100644 index 000000000..669e640fc --- /dev/null +++ b/src/test/subscription/testcase/skiplsn.sh @@ -0,0 +1,69 @@ +#!/bin/sh + +source $1/env_utils.sh $1 $2 + +case_db="skiplsn_db" + +function test_1() { + echo "create database and tables." + exec_sql $db $pub_node1_port "CREATE DATABASE $case_db" + exec_sql $db $sub_node1_port "CREATE DATABASE $case_db" + # Create some preexisting content on publisher + exec_sql $case_db $pub_node1_port "CREATE TABLE tab_rep (a int primary key, b int)" + + # Setup structure on subscriber + exec_sql $case_db $sub_node1_port "CREATE TABLE tab_rep (a int primary key, b int)" + + # Setup logical replication + echo "create publication and subscription." + publisher_connstr="port=$pub_node1_port host=$g_local_ip dbname=$case_db user=$username password=$passwd" + exec_sql $case_db $pub_node1_port "CREATE PUBLICATION tap_pub FOR ALL TABLES" + exec_sql $case_db $sub_node1_port "CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr' PUBLICATION tap_pub" + + # Wait for initial table sync to finish + wait_for_subscription_sync $case_db $sub_node1_port + + exec_sql $case_db $sub_node1_port "insert into tab_rep values (1,1)" + + logfile=$(get_log_file "sub_datanode1") + + location=$(awk 'END{print NR}' $logfile) + + exec_sql $case_db $pub_node1_port "insert into tab_rep values (1,2)" + + content=$(tail -n +$location $logfile) + commitlsn=$(expr "$content" : '.*commit_lsn:\s\([0-9|/|ABCDEF]*\).*') + + while [ -z $commitlsn ] + do + content=$(tail -n +$location $logfile) + commitlsn=$(expr "$content" : '.*commit_lsn:\s\([0-9|/|ABCDEF]*\).*') + done + + exec_sql $case_db $sub_node1_port "alter subscription tap_sub set (skiplsn = '$commitlsn')" + + exec_sql $case_db $pub_node1_port "insert into tab_rep values (2, 2)" + + wait_for_catchup $case_db $pub_node1_port "tap_sub" + + if [ "$(exec_sql $case_db $sub_node1_port "SELECT * FROM tab_rep")" = "1|1 +2|2" ]; then + echo "check data sync after skip conflict success" + else + echo "$failed_keyword when check data sync after skip conflict" + exit 1 + fi +} + +function tear_down() { + exec_sql $case_db $sub_node1_port "DROP SUBSCRIPTION IF EXISTS tap_sub" + exec_sql $case_db $pub_node1_port "DROP PUBLICATION IF EXISTS tap_pub" + + exec_sql $db $sub_node1_port "DROP DATABASE $case_db" + exec_sql $db $pub_node1_port "DROP DATABASE $case_db" + + echo "tear down" +} + +test_1 +tear_down \ No newline at end of file From 96942b2ca07f3de075a256e73c79aaa87bec1783 Mon Sep 17 00:00:00 2001 From: wuyuechuan Date: Mon, 14 Aug 2023 10:33:53 +0800 Subject: [PATCH 143/304] support pagerepair during primary startup --- .../cbb/grpc/remote_read_client.cpp | 47 ++++++++++++ .../storage/remote/remote_read.cpp | 76 ++++++++++++++++--- src/include/service/remote_read_client.h | 1 + 3 files changed, 112 insertions(+), 12 deletions(-) diff --git a/src/gausskernel/cbb/grpc/remote_read_client.cpp b/src/gausskernel/cbb/grpc/remote_read_client.cpp index a07b4cae7..0459d3129 100755 --- a/src/gausskernel/cbb/grpc/remote_read_client.cpp +++ b/src/gausskernel/cbb/grpc/remote_read_client.cpp @@ -238,6 +238,53 @@ int GetRemoteConnInfo(char* remoteAddress, char* remoteReadConnInfo, int len) return errCode; } +/** + * get connection + * @param remoteAddress ip@port + * @return connection if success or else nullptr + */ +static PGconn* RemoteGetConnection(char* remoteAddress) +{ + char remoteReadConnInfo[MAXPGPATH]; + int errCode = GetRemoteConnInfo(remoteAddress, remoteReadConnInfo, MAXPGPATH); + if (errCode != REMOTE_READ_OK) { + return nullptr; + } + PGconn* conGet = RemoteReadGetConn(remoteReadConnInfo); + if (conGet == nullptr) { + errCode = REMOTE_READ_RPC_ERROR; + return nullptr; + } + /* need to close by caller */ + return conGet; +} + +uint64 RemoteGetXlogReplayPtr(char* remoteAddress) +{ + PGconn* conGet = RemoteGetConnection(remoteAddress); + if (conGet == nullptr) { + return InvalidXLogRecPtr; + } + PGresult* res = PQexec(conGet, "SELECT lsn::varchar from pg_last_xlog_replay_location()"); + if (PQresultStatus(res) != PGRES_TUPLES_OK || PQgetisnull(res, 0, 0)) { + PQclear(res); + res = nullptr; + PQfinish(conGet); + conGet = nullptr; + return InvalidXLogRecPtr; + } + uint32 hi = 0; + uint32 lo = 0; + /* get remote lsn location */ + if (sscanf_s(PQgetvalue(res, 0, 0), "%X/%X", &hi, &lo) != 2) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("could not parse log location \"%s\"", PQgetvalue(res, 0, 0)))); + } + PQclear(res); + PQfinish(conGet); + return (((uint64)hi) << 32) | lo; +} + /* * @Description: remote read page diff --git a/src/gausskernel/storage/remote/remote_read.cpp b/src/gausskernel/storage/remote/remote_read.cpp index 505e159a7..1fd6dd115 100755 --- a/src/gausskernel/storage/remote/remote_read.cpp +++ b/src/gausskernel/storage/remote/remote_read.cpp @@ -30,6 +30,7 @@ #include "replication/walreceiver.h" #include "replication/walsender.h" #include "storage/remote_read.h" +#include "service/remote_read_client.h" #include /* @@ -102,6 +103,62 @@ void GetPrimaryServiceAddress(char *address, size_t address_len) SpinLockRelease(&walrcv->mutex); } + +static void FormatAddressByReplConn(replconninfo* replconninfo, char* remoteAddress, int addressLen) +{ + int rc = snprintf_s(remoteAddress, addressLen, (addressLen - 1), "%s@%d", + replconninfo->remotehost, + replconninfo->remoteport); + securec_check_ss(rc, "", ""); +} + +/** + * in startup, walsnder is not ready. we need to get remote address from replConnArray + * @param firstAddress first address + * @param secondAddress second address + * @param addressLen address length + */ +static void GetRemoteReadAddressFromReplconn(char* firstAddress, char* secondAddress, size_t addressLen) +{ + XLogRecPtr fastest_replay = InvalidXLogRecPtr; + XLogRecPtr second_fastest_replay = InvalidXLogRecPtr; + int fastest = 0; + int second_fastest = 0; + for (int i = 0; i < MAX_REPLNODE_NUM; i++) { + if (t_thrd.postmaster_cxt.ReplConnArray[i]) { + char remoteAddress[MAXPGPATH]; + FormatAddressByReplConn(t_thrd.postmaster_cxt.ReplConnArray[i], remoteAddress, MAXPGPATH); + XLogRecPtr insertXLogRecPtr = RemoteGetXlogReplayPtr(remoteAddress); + if (XLByteLT(second_fastest_replay, insertXLogRecPtr)) { + if (XLByteLT(fastest_replay, insertXLogRecPtr)) { + /* walsnd_replay is larger than fastest_replay */ + second_fastest = fastest; + second_fastest_replay = fastest_replay; + + fastest = i; + fastest_replay = insertXLogRecPtr; + } else { + /* walsnd_replay is in the range (second_fastest_replay, fastest_replay] */ + second_fastest = i; + second_fastest_replay = insertXLogRecPtr; + } + } + } + } + + /* find fastest replay standby */ + if (!XLogRecPtrIsInvalid(fastest_replay)) { + FormatAddressByReplConn(t_thrd.postmaster_cxt.ReplConnArray[fastest], firstAddress, MAXPGPATH); + + } + + /* find second fastest replay standby */ + if (!XLogRecPtrIsInvalid(second_fastest_replay)) { + FormatAddressByReplConn(t_thrd.postmaster_cxt.ReplConnArray[second_fastest], secondAddress, MAXPGPATH); + } + +} + /* * @Description: get remote address * @IN/OUT first_address: first address @@ -134,18 +191,13 @@ void GetRemoteReadAddress(char* firstAddress, char* secondAddress, size_t addres } } else if (IS_DN_MULTI_STANDYS_MODE()) { if (serverMode == PRIMARY_MODE) { - GetFastestReplayStandByServiceAddress(firstAddress, secondAddress, addressLen); - if (firstAddress[0] != '\0') { - GetIPAndPort(firstAddress, ip, port, MAX_IPADDR_LEN); - rc = snprintf_s(firstAddress, addressLen, (addressLen - 1), "%s@%s", ip, port); - securec_check_ss(rc, "", ""); - } - - if (secondAddress[0] != '\0') { - GetIPAndPort(secondAddress, ip, port, MAX_IPADDR_LEN); - rc = snprintf_s(secondAddress, addressLen, (addressLen - 1), "%s@%s", ip, port); - securec_check_ss(rc, "", ""); - } + /* address format: ip@port */ + if (RecoveryInProgress()) { + /* during recovery, walsnder is not valid so we cant get standby info from walsnder */ + GetRemoteReadAddressFromReplconn(firstAddress, secondAddress, addressLen); + } else { + GetFastestReplayStandByServiceAddress(firstAddress, secondAddress, addressLen); + } } else if (serverMode == STANDBY_MODE) { GetPrimaryServiceAddress(firstAddress, addressLen); if (firstAddress[0] != '\0') { diff --git a/src/include/service/remote_read_client.h b/src/include/service/remote_read_client.h index 032e119ec..9689bd324 100755 --- a/src/include/service/remote_read_client.h +++ b/src/include/service/remote_read_client.h @@ -43,6 +43,7 @@ extern int RemoteGetPage(char* remote_address, RepairBlockKey *key, uint32 block extern int RemoteGetFile(char* remoteAddress, RemoteReadFileKey *key, uint64 lsn, uint32 size, char* pageData, XLogRecPtr *remote_lsn, uint32 *remote_size, int timeout); extern int RemoteGetFileSize(char* remoteAddress, RemoteReadFileKey *key, uint64 lsn, int64 *size, int timeout); +extern uint64 RemoteGetXlogReplayPtr(char* remoteAddress); #endif /* REMOTE_READ_CLIENT_H */ From 19877e08bca6d1be0b3ada0427950053d12fee2f Mon Sep 17 00:00:00 2001 From: "arcoalien@qq.com" Date: Thu, 17 Aug 2023 14:55:26 +0800 Subject: [PATCH 144/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=A4=87=E6=9C=BA?= =?UTF-8?q?=E5=86=99=E8=BD=AC=E5=8F=91=E5=A4=84=E7=90=86gsql=20-c=E6=96=B9?= =?UTF-8?q?=E5=BC=8F=E6=89=A7=E8=A1=8C=E7=9A=84=E8=BF=9E=E7=BB=AD=E7=9A=84?= =?UTF-8?q?=E5=A4=9A=E4=B8=AAsql=E8=AF=AD=E5=8F=A5=E5=87=BA=E9=94=99?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/process/tcop/postgres.cpp | 18 ++++++++++++------ .../storage/replication/libpqsw.cpp | 3 +-- src/include/replication/libpqsw.h | 3 +-- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/gausskernel/process/tcop/postgres.cpp b/src/gausskernel/process/tcop/postgres.cpp index f05c29bf8..0d5096f36 100755 --- a/src/gausskernel/process/tcop/postgres.cpp +++ b/src/gausskernel/process/tcop/postgres.cpp @@ -2395,6 +2395,7 @@ static void exec_simple_query(const char* query_string, MessageType messageType, bool is_compl_sql = true; bool savedisAllowCommitRollback = false; bool needResetErrMsg = false; + const char* querystringForLibpqsw = NULL; /* * @hdfs * When messageType is 1, we get hybridmessage. This message @@ -2681,6 +2682,11 @@ static void exec_simple_query(const char* query_string, MessageType messageType, set_ps_display(commandTag, false); if (libpqsw_skip_check_readonly()) { + if (is_multistmt) { + querystringForLibpqsw = query_string_single[stmt_num - 1]; + } else { + querystringForLibpqsw = query_string; + } // create table as / select into / insert into if (nodeTag(parsetree) == T_CreateTableAsStmt || ((nodeTag(parsetree) == T_SelectStmt) && ((SelectStmt*)parsetree)->intoClause != NULL) @@ -2691,13 +2697,13 @@ static void exec_simple_query(const char* query_string, MessageType messageType, } if (libpqsw_get_redirect()) { - if (libpqsw_process_query_message(commandTag, NULL, query_string)) { - libpqsw_trace_q_msg(commandTag, query_string); + if (libpqsw_process_query_message(commandTag, NULL, querystringForLibpqsw)) { + libpqsw_trace_q_msg(commandTag, querystringForLibpqsw); if (snapshot_set) { PopActiveSnapshot(); } finish_xact_command(); - return; + continue; } } @@ -2812,8 +2818,8 @@ static void exec_simple_query(const char* query_string, MessageType messageType, break; } - if (libpqsw_process_query_message(commandTag, querytree_list, query_string, query_string_len)) { - libpqsw_trace_q_msg(commandTag, query_string); + if (libpqsw_process_query_message(commandTag, querytree_list, querystringForLibpqsw)) { + libpqsw_trace_q_msg(commandTag, querystringForLibpqsw); if (libpqsw_begin_command(commandTag) || libpqsw_end_command(commandTag)) { libpqsw_trace("libpq send sql at my side as well:%s", commandTag); } else { @@ -2823,7 +2829,7 @@ static void exec_simple_query(const char* query_string, MessageType messageType, CommandCounterIncrement(); finish_xact_command(); MemoryContextReset(OptimizerContext); - return; + continue; } } diff --git a/src/gausskernel/storage/replication/libpqsw.cpp b/src/gausskernel/storage/replication/libpqsw.cpp index 3cede836f..b08333ba9 100644 --- a/src/gausskernel/storage/replication/libpqsw.cpp +++ b/src/gausskernel/storage/replication/libpqsw.cpp @@ -1034,8 +1034,7 @@ bool libpqsw_process_parse_message(const char* commandTag, List* query_list) } /* process Q type msg, true if need in redirect mode*/ -bool libpqsw_process_query_message(const char* commandTag, List* query_list, const char* query_string, - size_t query_string_len) +bool libpqsw_process_query_message(const char* commandTag, List* query_list, const char* query_string) { if (IsAbortedTransactionBlockState()) { return false; diff --git a/src/include/replication/libpqsw.h b/src/include/replication/libpqsw.h index aa837ac84..8365743ff 100644 --- a/src/include/replication/libpqsw.h +++ b/src/include/replication/libpqsw.h @@ -46,8 +46,7 @@ bool libpqsw_process_message(int qtype, const StringInfo msg); /* process P type msg, true if need redirect*/ bool libpqsw_process_parse_message(const char* commandTag, List* query_list); /* process Q type msg, true if need in redirect mode*/ -bool libpqsw_process_query_message(const char* commandTag, List* query_list, const char* query_string, - size_t query_string_len = 0); +bool libpqsw_process_query_message(const char* commandTag, List* query_list, const char* query_string); /* is need send ready_for_query messge to front, if in redirect then false*/ bool libpqsw_need_end(); /* udpate if need ready_for_query messge flag */ From 6dd6d0e6e8bbf993e413e8a762aeee4b4ef7bd45 Mon Sep 17 00:00:00 2001 From: wuyuechuan Date: Thu, 17 Aug 2023 15:41:20 +0800 Subject: [PATCH 145/304] =?UTF-8?q?=E4=BD=BF=E7=94=A8sh=20build.sh=20-pkg?= =?UTF-8?q?=E6=89=93=E5=8C=85=E6=B7=BB=E5=8A=A0assessment=E6=8F=92?= =?UTF-8?q?=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build/script/aarch64_opengauss_list | 2 +- build/script/x86_64_opengauss_list | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build/script/aarch64_opengauss_list b/build/script/aarch64_opengauss_list index e776d1ee9..305cdd247 100644 --- a/build/script/aarch64_opengauss_list +++ b/build/script/aarch64_opengauss_list @@ -47,7 +47,7 @@ ./bin/gs_plan_simulator.sh ./bin/pg_xlogdump ./bin/pagehack -./bin/assessment_database +./bin/gs_assessment ./etc/kerberos/kadm5.acl ./etc/kerberos/kdc.conf ./etc/kerberos/krb5.conf diff --git a/build/script/x86_64_opengauss_list b/build/script/x86_64_opengauss_list index 5f2d0b1a5..69137a38f 100644 --- a/build/script/x86_64_opengauss_list +++ b/build/script/x86_64_opengauss_list @@ -47,7 +47,7 @@ ./bin/gs_plan_simulator.sh ./bin/pg_xlogdump ./bin/pagehack -./bin/assessment_database +./bin/gs_assessment ./etc/kerberos/kadm5.acl ./etc/kerberos/kdc.conf ./etc/kerberos/krb5.conf From 4481a3c920eafc461452b4468f304ff1f3084525 Mon Sep 17 00:00:00 2001 From: he-shaoyu Date: Thu, 17 Aug 2023 16:56:13 +0800 Subject: [PATCH 146/304] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E7=94=A8=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/test/subscription/schedule | 3 +- .../subscription/testcase/pub_subconflict.sh | 147 ++++++++++++++++++ 2 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 src/test/subscription/testcase/pub_subconflict.sh diff --git a/src/test/subscription/schedule b/src/test/subscription/schedule index 52ee04318..836ac0e58 100644 --- a/src/test/subscription/schedule +++ b/src/test/subscription/schedule @@ -12,4 +12,5 @@ ddl matviews change_wal_level skiplsn -disable \ No newline at end of file +disable +pub_subconflict \ No newline at end of file diff --git a/src/test/subscription/testcase/pub_subconflict.sh b/src/test/subscription/testcase/pub_subconflict.sh new file mode 100644 index 000000000..ee40c6b9d --- /dev/null +++ b/src/test/subscription/testcase/pub_subconflict.sh @@ -0,0 +1,147 @@ + +#!/bin/sh + +source $1/env_utils.sh $1 $2 + +case_db="conflict_db" + +function test_1() { + echo "create database and tables." + exec_sql $db $pub_node1_port "CREATE DATABASE $case_db" + exec_sql $db $sub_node1_port "CREATE DATABASE $case_db" + # Create some preexisting content on publisher + # => keep_local + exec_sql $case_db $sub_node1_port "ALTER SYSTEM SET subscription_conflict_resolution = keep_local" + + # Setup structure on subscriber + exec_sql $case_db $pub_node1_port "CREATE TABLE tab_keep (a int primary key, b char)" + exec_sql $case_db $sub_node1_port "CREATE TABLE tab_keep (a int primary key, b char)" + + # Setup logical replication + echo "create publication and subscription. (=> keep_local)" + publisher_connstr="port=$pub_node1_port host=$g_local_ip dbname=$case_db user=$username password=$passwd" + + exec_sql $case_db $pub_node1_port "CREATE PUBLICATION tap_pub1 FOR TABLE tab_keep" + exec_sql $case_db $sub_node1_port "CREATE SUBSCRIPTION tap_sub1 CONNECTION '$publisher_connstr' PUBLICATION tap_pub1" + + # Wait for initial table sync to finish + wait_for_subscription_sync $case_db $sub_node1_port + + exec_sql $case_db $sub_node1_port "INSERT INTO tab_keep VALUES(1, 'a')" + exec_sql $case_db $pub_node1_port "INSERT INTO tab_keep VALUES(1, 'b')" + + # Wait for catchup + wait_for_catchup $case_db $pub_node1_port "tap_sub1" + + if [ "$(exec_sql $case_db $sub_node1_port "SELECT count(*) FROM tab_keep where b = 'a'")" = "1" ]; then + echo "check pub_sub conflict for 1st sub success" + else + echo "$failed_keyword when check pub_sub conflict for 1st sub" + exit 1 + fi + + exec_sql $case_db $sub_node1_port "DROP SUBSCRIPTION tap_sub1" + exec_sql $case_db $pub_node1_port "DROP PUBLICATION tap_pub1" + + # => apply_remote + echo "create publication and subscription. (=> apply_remote)" + exec_sql $case_db $sub_node1_port "ALTER SYSTEM SET subscription_conflict_resolution = apply_remote" + + exec_sql $case_db $pub_node1_port "CREATE TABLE tab_apply (a int primary key, b char)" + exec_sql $case_db $sub_node1_port "CREATE TABLE tab_apply (a int primary key, b char)" + + exec_sql $case_db $pub_node1_port "CREATE PUBLICATION tap_pub2 FOR TABLE tab_apply" + exec_sql $case_db $sub_node1_port "CREATE SUBSCRIPTION tap_sub2 CONNECTION '$publisher_connstr' PUBLICATION tap_pub2" + + # Wait for initial table sync to finish + wait_for_subscription_sync $case_db $sub_node1_port + + exec_sql $case_db $sub_node1_port "INSERT INTO tab_apply VALUES(1, 'k')" + exec_sql $case_db $pub_node1_port "INSERT INTO tab_apply VALUES(1, 'a')" + + # Wait for catchup + wait_for_catchup $case_db $pub_node1_port "tap_sub2" + + + if [ "$(exec_sql $case_db $sub_node1_port "SELECT count(*) FROM tab_apply where b = 'a'")" = "1" ]; then + echo "check pub_sub conflict for 2nd sub success" + else + echo "$failed_keyword when check pub_sub conflict for 2nd sub" + exit 1 + fi + + exec_sql $case_db $sub_node1_port "DROP SUBSCRIPTION tap_sub2" + exec_sql $case_db $pub_node1_port "DROP PUBLICATION tap_pub2" + + # update apply_remote + echo "create publication and subscription. (update apply_remote)" + exec_sql $case_db $pub_node1_port "CREATE TABLE tab_updateApply (a int primary key, b char)" + exec_sql $case_db $sub_node1_port "CREATE TABLE tab_updateApply (a int primary key, b char)" + + exec_sql $case_db $pub_node1_port "CREATE PUBLICATION tap_pub3 FOR TABLE tab_updateApply" + exec_sql $case_db $sub_node1_port "CREATE SUBSCRIPTION tap_sub3 CONNECTION '$publisher_connstr' PUBLICATION tap_pub3" + + exec_sql $case_db $pub_node1_port "INSERT INTO tab_updateApply VALUES(1, 'a')" + + # Wait for catchup + wait_for_catchup $case_db $pub_node1_port "tap_sub3" + + exec_sql $case_db $sub_node1_port "INSERT INTO tab_updateApply VALUES(2, 'b')" + exec_sql $case_db $pub_node1_port "UPDATE tab_updateApply SET a = 2 where a = 1" + + # Wait for catchup + wait_for_catchup $case_db $pub_node1_port "tap_sub3" + + if [ "$(exec_sql $case_db $sub_node1_port "SELECT * FROM tab_updateApply")" = "2|a" ]; then + echo "check pub_sub conflict for 3rd sub success" + else + echo "$failed_keyword when check pub_sub conflict for 3rd sub" + exit 1 + fi + + exec_sql $case_db $sub_node1_port "DROP SUBSCRIPTION tap_sub3" + exec_sql $case_db $pub_node1_port "DROP PUBLICATION tap_pub3" + + #unique conflict + echo "create publication and subscription. (unique conflict)" + exec_sql $case_db $pub_node1_port "CREATE TABLE tab_unique (a int primary key, b char, c int unique)" + exec_sql $case_db $sub_node1_port "CREATE TABLE tab_unique (a int primary key, b char, c int unique)" + + exec_sql $case_db $pub_node1_port "CREATE PUBLICATION tap_pub4 FOR TABLE tab_unique" + exec_sql $case_db $sub_node1_port "CREATE SUBSCRIPTION tap_sub4 CONNECTION '$publisher_connstr' PUBLICATION tap_pub4" + + exec_sql $case_db $sub_node1_port "INSERT INTO tab_unique VALUES (1, 'a', 1), (2, 'c', 2), (3, 'c', 3)" + + # Wait for catchup + wait_for_catchup $case_db $pub_node1_port "tap_sub4" + + exec_sql $case_db $pub_node1_port "INSERT INTO tab_unique VALUES(3, 'k', 4)" + + wait_for_catchup $case_db $pub_node1_port "tap_sub4" + + exec_sql $case_db $pub_node1_port "INSERT INTO tab_unique VALUES(1, 'b', 2)" + + if [ "$(exec_sql $case_db $sub_node1_port "SELECT * FROM tab_unique")" = "1|a|1 +2|c|2 +3|k|4" ]; then + echo "check pub_sub conflict for 4th sub success" + else + echo "$failed_keyword when check pub_sub conflict for 4th sub" + exit 1 + fi + + exec_sql $case_db $sub_node1_port "DROP SUBSCRIPTION tap_sub4" + exec_sql $case_db $pub_node1_port "DROP PUBLICATION tap_pub4" +} + +function tear_down(){ + exec_sql $case_db $sub_node1_port "ALTER SYSTEM SET subscription_conflict_resolution = error" + + exec_sql $db $sub_node1_port "DROP DATABASE $case_db" + exec_sql $db $pub_node1_port "DROP DATABASE $case_db" + + echo "tear down" +} + +test_1 +tear_down \ No newline at end of file From b6d22a39205fd061fb580d8323db065c5b0a481f Mon Sep 17 00:00:00 2001 From: suncan <1006949218@qq.com> Date: Tue, 4 Jul 2023 14:29:05 +0800 Subject: [PATCH 147/304] =?UTF-8?q?=E6=94=AF=E6=8C=81=E6=9E=9A=E4=B8=BE?= =?UTF-8?q?=E7=B1=BB=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/catalog/pg_enum.cpp | 3 ++- src/common/backend/catalog/pg_set.cpp | 23 -------------------- src/common/backend/parser/parse_collate.cpp | 24 +++++++++++++++++++++ src/include/catalog/pg_enum.h | 2 +- src/include/utils/builtins.h | 1 + 5 files changed, 28 insertions(+), 25 deletions(-) diff --git a/src/common/backend/catalog/pg_enum.cpp b/src/common/backend/catalog/pg_enum.cpp index e8ede4323..cd5e034f6 100644 --- a/src/common/backend/catalog/pg_enum.cpp +++ b/src/common/backend/catalog/pg_enum.cpp @@ -47,7 +47,7 @@ static int sort_order_cmp(const void* p1, const void* p2); * * vals is a list of Value strings. */ -void EnumValuesCreate(Oid enumTypeOid, List* vals) +void EnumValuesCreate(Oid enumTypeOid, List* vals, Oid collation) { Relation pg_enum = NULL; NameData enumlabel; @@ -60,6 +60,7 @@ void EnumValuesCreate(Oid enumTypeOid, List* vals) HeapTuple tup = NULL; num_elems = list_length(vals); + check_duplicate_value_by_collation(vals, collation, TYPTYPE_ENUM); /* * We do not bother to check the list of values for duplicates --- if you diff --git a/src/common/backend/catalog/pg_set.cpp b/src/common/backend/catalog/pg_set.cpp index 79335c9c0..cf3e7e15d 100644 --- a/src/common/backend/catalog/pg_set.cpp +++ b/src/common/backend/catalog/pg_set.cpp @@ -58,29 +58,6 @@ static void checkSetLableValue(char *label) } } -void check_duplicate_value_by_collation(List* vals, Oid collation, char type) -{ - if (!is_b_format_collation(collation)) { - return ; - } - - ListCell* lc = NULL; - foreach (lc, vals) { - ListCell* next_cell = lc->next; - char* lab = strVal(lfirst(lc)); - while(next_cell != NULL) { - char* next_lab = strVal(lfirst(next_cell)); - if (varstr_cmp_by_builtin_collations(lab, strlen(lab), next_lab, strlen(next_lab), collation) == 0) { - const char* type_name = NULL; - type_name = (type == TYPTYPE_SET) ? "set" : "enum"; - ereport(ERROR, (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), - errmsg("%s has duplicate key value \"%s\" = \"%s\"", type_name, lab, next_lab))); - } - next_cell = lnext(next_cell); - } - } -} - /* * SetValuesCreate * Create an entry in pg_set for each of the supplied set values. diff --git a/src/common/backend/parser/parse_collate.cpp b/src/common/backend/parser/parse_collate.cpp index 624fcef3c..291e0dec0 100644 --- a/src/common/backend/parser/parse_collate.cpp +++ b/src/common/backend/parser/parse_collate.cpp @@ -44,6 +44,7 @@ #include "catalog/pg_aggregate.h" #include "catalog/pg_collation.h" #include "catalog/pg_proc.h" +#include "catalog/gs_collation.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "parser/parse_collate.h" @@ -1342,3 +1343,26 @@ static void assign_expression_charset(Node* node, Oid target_collation) return; } + +void check_duplicate_value_by_collation(List* vals, Oid collation, char type) +{ + if (!is_b_format_collation(collation)) { + return ; + } + + ListCell* lc = NULL; + foreach (lc, vals) { + ListCell* next_cell = lc->next; + char* lab = strVal(lfirst(lc)); + while(next_cell != NULL) { + char* next_lab = strVal(lfirst(next_cell)); + if (varstr_cmp_by_builtin_collations(lab, strlen(lab), next_lab, strlen(next_lab), collation) == 0) { + const char* type_name = NULL; + type_name = (type == TYPTYPE_SET) ? "set" : "enum"; + ereport(ERROR, (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), + errmsg("%s has duplicate key value \"%s\" = \"%s\"", type_name, lab, next_lab))); + } + next_cell = lnext(next_cell); + } + } +} \ No newline at end of file diff --git a/src/include/catalog/pg_enum.h b/src/include/catalog/pg_enum.h index dd2e1c7c3..d692ff0b6 100644 --- a/src/include/catalog/pg_enum.h +++ b/src/include/catalog/pg_enum.h @@ -63,7 +63,7 @@ typedef FormData_pg_enum *Form_pg_enum; /* * prototypes for functions in pg_enum.c */ -extern void EnumValuesCreate(Oid enumTypeOid, List *vals); +extern void EnumValuesCreate(Oid enumTypeOid, List *vals, Oid collation = InvalidOid); extern void EnumValuesDelete(Oid enumTypeOid); extern void AddEnumLabel(Oid enumTypeOid, const char *newVal, const char *neighbor, bool newValIsAfter, diff --git a/src/include/utils/builtins.h b/src/include/utils/builtins.h index c01264ee0..ad380a880 100644 --- a/src/include/utils/builtins.h +++ b/src/include/utils/builtins.h @@ -279,6 +279,7 @@ extern Datum int4setle(PG_FUNCTION_ARGS); extern Datum int4setge(PG_FUNCTION_ARGS); extern Datum findinset(PG_FUNCTION_ARGS); extern Datum btint8sortsupport(PG_FUNCTION_ARGS); +extern void check_duplicate_value_by_collation(List* vals, Oid collation, char type); /* int.c */ extern Datum int2in(PG_FUNCTION_ARGS); From f267f519bb9a4c0f6f3653cca4fd533b28087b31 Mon Sep 17 00:00:00 2001 From: movead Date: Thu, 17 Aug 2023 12:22:22 +0800 Subject: [PATCH 148/304] fix wait condition while redo vacuum wal record --- src/bin/gs_guc/cluster_guc.conf | 2 +- src/common/backend/utils/misc/guc/guc_storage.cpp | 4 ++-- .../storage/access/transam/extreme_rto/dispatcher.cpp | 2 +- .../access/transam/parallel_recovery/dispatcher.cpp | 2 +- .../storage/access/transam/parallel_recovery/page_redo.cpp | 7 +++++-- src/include/access/multi_redo_api.h | 6 +++--- src/include/knl/knl_guc/knl_instance_attr_storage.h | 2 +- src/test/regress/output/recovery_2pc_tools.source | 2 +- 8 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/bin/gs_guc/cluster_guc.conf b/src/bin/gs_guc/cluster_guc.conf index 7ce0dfd97..c7874de7a 100755 --- a/src/bin/gs_guc/cluster_guc.conf +++ b/src/bin/gs_guc/cluster_guc.conf @@ -33,7 +33,7 @@ alarm_component|string|0,0|NULL|NULL| alarm_report_interval|int|0,2147483647|NULL|NULL| allow_concurrent_tuple_update|bool|0,0|NULL|NULL| enable_huge_pages|bool|0,0|NULL|NULL| -parallel_recovery_cost_record|bool|0,0|NULL|NULL| +enable_time_report|bool|0,0|NULL|NULL| enable_batch_dispatch|bool|0,0|NULL|NULL| allow_create_sysobject|bool|0,0|NULL|NULL| allow_system_table_mods|bool|0,0|NULL|NULL| diff --git a/src/common/backend/utils/misc/guc/guc_storage.cpp b/src/common/backend/utils/misc/guc/guc_storage.cpp index 83458d216..16a6b7174 100755 --- a/src/common/backend/utils/misc/guc/guc_storage.cpp +++ b/src/common/backend/utils/misc/guc/guc_storage.cpp @@ -1222,13 +1222,13 @@ static void InitStorageConfigureNamesBool() NULL, NULL}, - {{"parallel_recovery_cost_record", + {{"enable_time_report", PGC_POSTMASTER, NODE_SINGLENODE, RESOURCES_RECOVERY, gettext_noop("Record process time in every stage of parallel reovery"), NULL}, - &g_instance.attr.attr_storage.parallel_recovery_cost_record, + &g_instance.attr.attr_storage.enable_time_report, false, NULL, NULL, diff --git a/src/gausskernel/storage/access/transam/extreme_rto/dispatcher.cpp b/src/gausskernel/storage/access/transam/extreme_rto/dispatcher.cpp index a881731e1..9a1daeb2b 100755 --- a/src/gausskernel/storage/access/transam/extreme_rto/dispatcher.cpp +++ b/src/gausskernel/storage/access/transam/extreme_rto/dispatcher.cpp @@ -2138,7 +2138,7 @@ void redo_get_worker_time_count(RedoWorkerTimeCountsInfo **workerCountInfoList, knl_parallel_redo_state state = g_instance.comm_cxt.predo_cxt.state; SpinLockRelease(&(g_instance.comm_cxt.predo_cxt.rwlock)); - if (state != REDO_IN_PROGRESS || !g_instance.attr.attr_storage.parallel_recovery_cost_record) { + if (state != REDO_IN_PROGRESS || !g_instance.attr.attr_storage.enable_time_report) { *realNum = 0; return; } diff --git a/src/gausskernel/storage/access/transam/parallel_recovery/dispatcher.cpp b/src/gausskernel/storage/access/transam/parallel_recovery/dispatcher.cpp index a77019908..7c5af05b5 100755 --- a/src/gausskernel/storage/access/transam/parallel_recovery/dispatcher.cpp +++ b/src/gausskernel/storage/access/transam/parallel_recovery/dispatcher.cpp @@ -2008,7 +2008,7 @@ void redo_get_worker_time_count(RedoWorkerTimeCountsInfo **workerCountInfoList, knl_parallel_redo_state state = g_instance.comm_cxt.predo_cxt.state; SpinLockRelease(&(g_instance.comm_cxt.predo_cxt.rwlock)); - if (state != REDO_IN_PROGRESS || !g_instance.attr.attr_storage.parallel_recovery_cost_record) { + if (state != REDO_IN_PROGRESS || !g_instance.attr.attr_storage.enable_time_report) { *realNum = 0; return; } diff --git a/src/gausskernel/storage/access/transam/parallel_recovery/page_redo.cpp b/src/gausskernel/storage/access/transam/parallel_recovery/page_redo.cpp index b49b081e8..de16d88d1 100755 --- a/src/gausskernel/storage/access/transam/parallel_recovery/page_redo.cpp +++ b/src/gausskernel/storage/access/transam/parallel_recovery/page_redo.cpp @@ -565,6 +565,7 @@ static void wait_valid_snapshot(XLogReaderState *record) uint8 info = (XLogRecGetInfo(record) & ~XLR_INFO_MASK) & XLOG_HEAP_OPMASK; xl_heap_clean* xlrec = NULL; uint64 blockcnt = 0; + XLogRecPtr cur_transed_lsn = InvalidXLogRecPtr; if(rm_id != RM_HEAP2_ID || info != XLOG_HEAP2_CLEAN) return; @@ -577,14 +578,16 @@ static void wait_valid_snapshot(XLogReaderState *record) */ while(t_thrd.xact_cxt.ShmemVariableCache->standbyXmin < xlrec->latestRemovedXid && !in_full_sync_dispatch()) { + if(cur_transed_lsn == InvalidXLogRecPtr) + cur_transed_lsn = getTransedTxnLsn(g_dispatcher->txnWorker); /* * Normaly, it need wait for startup thread handle xact wal records, but there be a case * that if a very old xid commit and no new xact comes then xlrec->latestRemovedXid > * t_thrd.xact_cxt.ShmemVariableCache->standbyXmin all the time. * - * So if we do not have new xact work in startup thread, it avoid wait. + * So if all xact record before current vacuum record finished, then avoid wait. */ - if (getTransedTxnLsn(g_dispatcher->txnWorker) < record->EndRecPtr) + if (cur_transed_lsn <= GetXLogReplayRecPtr(NULL)) return; pg_usleep(10); blockcnt++; diff --git a/src/include/access/multi_redo_api.h b/src/include/access/multi_redo_api.h index eccb4e6b0..750e5b451 100644 --- a/src/include/access/multi_redo_api.h +++ b/src/include/access/multi_redo_api.h @@ -127,14 +127,14 @@ void ResetXLogStatics(); static inline void GetRedoStartTime(RedoTimeCost &cost) { - if(!g_instance.attr.attr_storage.parallel_recovery_cost_record) + if(!g_instance.attr.attr_storage.enable_time_report) return; cost.startTime = GetCurrentTimestamp(); } static inline void CountRedoTime(RedoTimeCost &cost) { - if(!g_instance.attr.attr_storage.parallel_recovery_cost_record) + if(!g_instance.attr.attr_storage.enable_time_report) return; cost.totalDuration += GetCurrentTimestamp() - cost.startTime; cost.counter += 1; @@ -144,7 +144,7 @@ static inline void CountAndGetRedoTime(RedoTimeCost &curCost, RedoTimeCost &next { uint64 curTime = 0; - if(!g_instance.attr.attr_storage.parallel_recovery_cost_record) + if(!g_instance.attr.attr_storage.enable_time_report) return; curTime = GetCurrentTimestamp(); diff --git a/src/include/knl/knl_guc/knl_instance_attr_storage.h b/src/include/knl/knl_guc/knl_instance_attr_storage.h index b3613456b..7161e8d11 100755 --- a/src/include/knl/knl_guc/knl_instance_attr_storage.h +++ b/src/include/knl/knl_guc/knl_instance_attr_storage.h @@ -213,7 +213,7 @@ typedef struct knl_instance_attr_storage { #endif bool enable_huge_pages; int huge_page_size; - bool parallel_recovery_cost_record; + bool enable_time_report; bool enable_batch_dispatch; int parallel_recovery_timeout; int parallel_recovery_batch; diff --git a/src/test/regress/output/recovery_2pc_tools.source b/src/test/regress/output/recovery_2pc_tools.source index 3a1583490..9c4c59000 100644 --- a/src/test/regress/output/recovery_2pc_tools.source +++ b/src/test/regress/output/recovery_2pc_tools.source @@ -332,6 +332,7 @@ select name,vartype,unit,min_val,max_val from pg_settings where name <> 'qunit_c enable_tde | bool | | | enable_thread_pool | bool | | | enable_tidscan | bool | | | + enable_time_report | bool | | | enable_union_all_subquery_orderby | bool | | | enable_upgrade_merge_lock_mode | bool | | | enable_user_metric_persistent | bool | | | @@ -515,7 +516,6 @@ select name,vartype,unit,min_val,max_val from pg_settings where name <> 'qunit_c pagewriter_sleep | integer | ms | 0 | 3600000 pagewriter_thread_num | integer | | 1 | 16 parallel_recovery_batch | integer | | 1 | 100000 - parallel_recovery_cost_record | bool | | | parallel_recovery_timeout | integer | ms | 1 | 1000 partition_iterator_elimination | bool | | | partition_lock_upgrade_timeout | integer | | -1 | 3000 From 401b50012d526a94adfc008ed26ecc572182c285 Mon Sep 17 00:00:00 2001 From: chenxiaobin19 <1025221611@qq.com> Date: Fri, 18 Aug 2023 15:17:39 +0800 Subject: [PATCH 149/304] =?UTF-8?q?=E6=A3=80=E8=A7=86=E6=84=8F=E8=A7=81?= =?UTF-8?q?=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../optimizer/commands/subscriptioncmds.cpp | 4 +- .../process/threadpool/knl_instance.cpp | 2 + .../storage/replication/logical/launcher.cpp | 4 + .../storage/replication/logical/worker.cpp | 94 +++++++++++++++---- src/include/knl/knl_instance.h | 3 + src/include/replication/worker_internal.h | 2 + 6 files changed, 88 insertions(+), 21 deletions(-) diff --git a/src/gausskernel/optimizer/commands/subscriptioncmds.cpp b/src/gausskernel/optimizer/commands/subscriptioncmds.cpp index aa52f2ebe..e4da27801 100644 --- a/src/gausskernel/optimizer/commands/subscriptioncmds.cpp +++ b/src/gausskernel/optimizer/commands/subscriptioncmds.cpp @@ -195,7 +195,7 @@ static void parse_subscription_options(const List *stmt_options, bits32 supporte errmsg("conflicting or redundant options"))); } - opts->specified_opts = SUBOPT_CONNECT; + opts->specified_opts |= SUBOPT_CONNECT; opts->connect = defGetBoolean(defel); } else if (IsSet(supported_opts, SUBOPT_SKIPLSN) && strcmp(defel->defname, "skiplsn") == 0) { if (IsSet(opts->specified_opts, SUBOPT_SKIPLSN)) { @@ -205,7 +205,7 @@ static void parse_subscription_options(const List *stmt_options, bits32 supporte } char *lsn_str = defGetString(defel); - opts->specified_opts = SUBOPT_SKIPLSN; + opts->specified_opts |= SUBOPT_SKIPLSN; /* Setting lsn = 'NONE' is treated as resetting LSN */ if (strcmp(lsn_str, "none") == 0) { opts->skiplsn = InvalidXLogRecPtr; diff --git a/src/gausskernel/process/threadpool/knl_instance.cpp b/src/gausskernel/process/threadpool/knl_instance.cpp index 7d853aca8..216f6f21d 100755 --- a/src/gausskernel/process/threadpool/knl_instance.cpp +++ b/src/gausskernel/process/threadpool/knl_instance.cpp @@ -1016,6 +1016,8 @@ void knl_instance_init() for (int i = 0; i < DB_CMPT_MAX; i++) { pthread_mutex_init(&g_instance.loadPluginLock[i], NULL); } + g_instance.needCheckConflictSubIds = NIL; + pthread_mutex_init(&g_instance.subIdsLock, NULL); #endif knl_g_datadir_init(&g_instance.datadir_cxt); diff --git a/src/gausskernel/storage/replication/logical/launcher.cpp b/src/gausskernel/storage/replication/logical/launcher.cpp index 9bffe943b..1f136a7fa 100644 --- a/src/gausskernel/storage/replication/logical/launcher.cpp +++ b/src/gausskernel/storage/replication/logical/launcher.cpp @@ -286,6 +286,10 @@ void logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname, Oid user TIMESTAMP_NOBEGIN(worker->reply_time); worker->workerLaunchTime = GetCurrentTimestamp(); + pthread_mutex_lock(&g_instance.subIdsLock); + worker->needCheckConflict = list_member_oid(g_instance.needCheckConflictSubIds, subid); + pthread_mutex_unlock(&g_instance.subIdsLock); + t_thrd.applylauncher_cxt.applyLauncherShm->startingWorker = worker; LWLockRelease(LogicalRepWorkerLock); diff --git a/src/gausskernel/storage/replication/logical/worker.cpp b/src/gausskernel/storage/replication/logical/worker.cpp index 167333633..54a903fe0 100644 --- a/src/gausskernel/storage/replication/logical/worker.cpp +++ b/src/gausskernel/storage/replication/logical/worker.cpp @@ -516,6 +516,17 @@ static void apply_handle_commit(StringInfo s) /* Process any tables that are being synchronized in parallel. */ process_syncing_tables(commit_data.end_lsn); + + if (t_thrd.applyworker_cxt.curWorker->needCheckConflict) { + t_thrd.applyworker_cxt.curWorker->needCheckConflict = false; + MemoryContext oldctx = MemoryContextSwitchTo(INSTANCE_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_DEFAULT)); + pthread_mutex_lock(&g_instance.subIdsLock); + g_instance.needCheckConflictSubIds = list_delete_oid(g_instance.needCheckConflictSubIds, + t_thrd.applyworker_cxt.curWorker->subid); + pthread_mutex_unlock(&g_instance.subIdsLock); + MemoryContextSwitchTo(oldctx); + } + if (t_thrd.applyworker_cxt.isSkipTransaction) { StopSkippingChanges(); } @@ -596,16 +607,19 @@ Oid find_conflict_tuple(EState *estate, TupleTableSlot *remoteslot, TupleTableSl /* Check the replica identity index first */ replidxoid = RelationGetReplicaIndex(relinfo->ri_RelationDesc); - found = RelationFindReplTuple(estate, relinfo->ri_RelationDesc, replidxoid, LockTupleExclusive, remoteslot, - localslot, fakeRelInfo); - if (found) { - if (originslot != NULL && ItemPointerCompare(tableam_tops_get_t_self(relinfo->ri_RelationDesc, - localslot->tts_tuple), tableam_tops_get_t_self(relinfo->ri_RelationDesc, - originslot->tts_tuple)) == 0) { - /* If conflict with the tuple to be updated, ignore it. */ - found = false; - } else { - return replidxoid; + if (OidIsValid(replidxoid)) { + found = RelationFindReplTuple(estate, relinfo->ri_RelationDesc, replidxoid, LockTupleExclusive, remoteslot, + localslot, fakeRelInfo); + + if (found) { + if (originslot != NULL && ItemPointerCompare(tableam_tops_get_t_self(relinfo->ri_RelationDesc, + localslot->tts_tuple), tableam_tops_get_t_self(relinfo->ri_RelationDesc, + originslot->tts_tuple)) == 0) { + /* If conflict with the tuple to be updated, ignore it. */ + found = false; + } else { + return replidxoid; + } } } @@ -693,7 +707,8 @@ static void apply_handle_insert(StringInfo s) ExecOpenIndices(estate->es_result_relation_info, false); - if ((conflictIndexOid = find_conflict_tuple(estate, remoteslot, localslot, &fakeRelInfo)) != InvalidOid) { + if (t_thrd.applyworker_cxt.curWorker->needCheckConflict && + (conflictIndexOid = find_conflict_tuple(estate, remoteslot, localslot, &fakeRelInfo)) != InvalidOid) { StringInfoData localtup, remotetup; initStringInfo(&localtup); tuple_to_stringinfo(rel->localrel, &localtup, RelationGetDescr(rel->localrel), @@ -743,11 +758,31 @@ static void apply_handle_insert(StringInfo s) FreeStringInfo(&localtup); FreeStringInfo(&remotetup); } else { - /* Get fake relation and partition for patitioned table */ - GetFakeRelAndPart(estate, rel->localrel, remoteslot, &fakeRelInfo); + PG_TRY(); + { + /* Get fake relation and partition for patitioned table */ + GetFakeRelAndPart(estate, rel->localrel, remoteslot, &fakeRelInfo); - /* Do the insert. */ - ExecSimpleRelationInsert(estate, remoteslot, &fakeRelInfo); + /* Do the insert. */ + ExecSimpleRelationInsert(estate, remoteslot, &fakeRelInfo); + } + PG_CATCH(); + { + ErrorData* errdata = NULL; + + (void*)MemoryContextSwitchTo(oldctx); + errdata = CopyErrorData(); + if (errdata->sqlerrcode == ERRCODE_UNIQUE_VIOLATION) { + (void*)MemoryContextSwitchTo(INSTANCE_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_DEFAULT)); + pthread_mutex_lock(&g_instance.subIdsLock); + g_instance.needCheckConflictSubIds = lappend_oid(g_instance.needCheckConflictSubIds, + t_thrd.applyworker_cxt.curWorker->subid); + pthread_mutex_unlock(&g_instance.subIdsLock); + (void*)MemoryContextSwitchTo(oldctx); + } + PG_RE_THROW(); + } + PG_END_TRY(); } /* Cleanup. */ @@ -924,7 +959,8 @@ static void apply_handle_update(StringInfo s) slot_modify_data(remoteslot, localslot, rel, &newtup); MemoryContextSwitchTo(oldctx); - if ((conflictIndexOid = find_conflict_tuple(estate, remoteslot, conflictLocalSlot, &fakeRelInfo, localslot)) + if (t_thrd.applyworker_cxt.curWorker->needCheckConflict && + (conflictIndexOid = find_conflict_tuple(estate, remoteslot, conflictLocalSlot, &fakeRelInfo, localslot)) != InvalidOid) { StringInfoData localtup, remotetup; initStringInfo(&localtup); @@ -977,10 +1013,30 @@ static void apply_handle_update(StringInfo s) FreeStringInfo(&localtup); FreeStringInfo(&remotetup); } else { - EvalPlanQualSetSlot(&epqstate, remoteslot); + PG_TRY(); + { + EvalPlanQualSetSlot(&epqstate, remoteslot); - /* Do the actual update. */ - ExecSimpleRelationUpdate(estate, &epqstate, localslot, remoteslot, &fakeRelInfo); + /* Do the actual update. */ + ExecSimpleRelationUpdate(estate, &epqstate, localslot, remoteslot, &fakeRelInfo); + } + PG_CATCH(); + { + ErrorData* errdata = NULL; + + (void*)MemoryContextSwitchTo(oldctx); + errdata = CopyErrorData(); + if (errdata->sqlerrcode == ERRCODE_UNIQUE_VIOLATION) { + (void*)MemoryContextSwitchTo(INSTANCE_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_DEFAULT)); + pthread_mutex_lock(&g_instance.subIdsLock); + g_instance.needCheckConflictSubIds = lappend_oid(g_instance.needCheckConflictSubIds, + t_thrd.applyworker_cxt.curWorker->subid); + pthread_mutex_unlock(&g_instance.subIdsLock); + (void*)MemoryContextSwitchTo(oldctx); + } + PG_RE_THROW(); + } + PG_END_TRY(); } } else { /* diff --git a/src/include/knl/knl_instance.h b/src/include/knl/knl_instance.h index 674e8eab9..1853d7035 100755 --- a/src/include/knl/knl_instance.h +++ b/src/include/knl/knl_instance.h @@ -1354,6 +1354,9 @@ typedef struct knl_instance_context { void *raw_parser_hook[DB_CMPT_MAX]; char *llvmIrFilePath[DB_CMPT_MAX]; pthread_mutex_t loadPluginLock[DB_CMPT_MAX]; + + List* needCheckConflictSubIds; + pthread_mutex_t subIdsLock; #endif pg_atomic_uint32 extensionNum; knl_g_audit_context audit_cxt; diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h index 4642c343e..7ac83f729 100644 --- a/src/include/replication/worker_internal.h +++ b/src/include/replication/worker_internal.h @@ -46,6 +46,8 @@ typedef struct LogicalRepWorker TimestampTz last_recv_time; XLogRecPtr reply_lsn; TimestampTz reply_time; + + bool needCheckConflict; } LogicalRepWorker; typedef struct ApplyLauncherShmStruct { From 75717a617202c487b9a64553c0355e0d0dceec75 Mon Sep 17 00:00:00 2001 From: zhangao_za <18829237393@163.com> Date: Mon, 21 Aug 2023 19:27:30 +0800 Subject: [PATCH 150/304] =?UTF-8?q?pg=5Fcontrol=20ss=20=E6=9B=B4=E6=96=B0?= =?UTF-8?q?=E6=89=80=E6=9C=89=E8=8A=82=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/pg_ctl/backup.cpp | 33 ++++++++++++++++++++++++--------- src/bin/pg_ctl/pg_ctl.cpp | 33 ++++++++++++++++++++++++++++++++- src/common/port/tool_common.cpp | 1 + src/include/tool_common.h | 1 + 4 files changed, 58 insertions(+), 10 deletions(-) diff --git a/src/bin/pg_ctl/backup.cpp b/src/bin/pg_ctl/backup.cpp index 9ccd46d89..377176f9d 100755 --- a/src/bin/pg_ctl/backup.cpp +++ b/src/bin/pg_ctl/backup.cpp @@ -775,6 +775,8 @@ static bool ReceiveAndUnpackTarFile(PGconn* conn, PGresult* res, int rownum) struct stat st; int nRet = 0; bool forbid_write = false; + char pg_control_file[MAXPGPATH] = {0}; + errno_t errorno = EOK; if (!GetCurrentPath(current_path, res, rownum)) { return false; @@ -792,6 +794,12 @@ static bool ReceiveAndUnpackTarFile(PGconn* conn, PGresult* res, int rownum) } PQclear(res); + if (ss_instance_config.dss.enable_dss) { + errorno = snprintf_s(pg_control_file, MAXPGPATH, MAXPGPATH - 1, "%s/pg_control", + ss_instance_config.dss.vgname); + securec_check_ss_c(errorno, "\0", "\0"); + } + while (1) { int r; @@ -1013,22 +1021,29 @@ static bool ReceiveAndUnpackTarFile(PGconn* conn, PGresult* res, int rownum) continue; } - /* pg_control will be written into a specified postion of main stanby corresponding to */ - if (ss_instance_config.dss.enable_dss && strcmp(filename, "+data/pg_control") == 0) { + /* pg_control will be written into pages of each interconnect nodes in stanby cluster corresponding to */ + if (ss_instance_config.dss.enable_dss && strcmp(filename, pg_control_file) == 0) { pg_log(PG_WARNING, _("file size %d. \n"), r); - int main_standby_id = ss_instance_config.dss.instance_id; - off_t seekpos = (off_t)BLCKSZ * main_standby_id; - fseek(file, seekpos, SEEK_SET); - } - - if (forbid_write == false) { - if (fwrite(copybuf, r, 1, file) != 1) { + int node; + for (node = 0; node < ss_instance_config.dss.interNodeNum; node++) { + off_t seekpos = (off_t)BLCKSZ * node; + fseek(file, seekpos, SEEK_SET); + if (fwrite(copybuf, r, 1, file) != 1) { + pg_log(PG_WARNING, _("could not write to file \"%s\": %s\n"), filename, strerror(errno)); + DisconnectConnection(); + FREE_AND_RESET(copybuf); + return false; + } + } + } else { + if (!forbid_write && fwrite(copybuf, r, 1, file) != 1) { pg_log(PG_WARNING, _("could not write to file \"%s\": %s\n"), filename, strerror(errno)); DisconnectConnection(); FREE_AND_RESET(copybuf); return false; } } + totaldone += r; if (showprogress && build_mode != COPY_SECURE_FILES_BUILD) { progress_report(rownum, filename, false); diff --git a/src/bin/pg_ctl/pg_ctl.cpp b/src/bin/pg_ctl/pg_ctl.cpp index 3984fef1a..4f23de9b9 100755 --- a/src/bin/pg_ctl/pg_ctl.cpp +++ b/src/bin/pg_ctl/pg_ctl.cpp @@ -223,6 +223,7 @@ static char* new_role = "passive"; static char start_minority_file[MAXPGPATH]; static int get_instance_id(void); static int ss_get_primary_id(void); +static int ss_get_inter_node_nums(const char* interconn_url); bool ss_read_config(void); static unsigned int vote_num = 0; static unsigned int xmode = 2; @@ -6852,7 +6853,7 @@ int main(int argc, char** argv) FREE_AND_RESET(vgname); FREE_AND_RESET(vgdata); - FREE_AND_RESET(vgdata); + FREE_AND_RESET(vglog); parse_vgname_args(optarg); ss_instance_config.dss.vgname = xstrdup(vgname); ss_instance_config.dss.vgdata = xstrdup(vgdata); @@ -7450,6 +7451,32 @@ static int ss_get_primary_id(void) return reformerCtrl->primaryInstId; } +static int ss_get_inter_node_nums(const char* interconn_url) +{ + errno_t rc; + int nodeNum = 0; + char* next_token = NULL; + char* token = NULL; + const char* delim = ","; + if (interconn_url == NULL || interconn_url[0] == '\0') { + pg_log(PG_WARNING, _("can not contain interconnect nodes.\n")); + return nodeNum; + } + + char* strs = (char*)palloc(strlen(interconn_url) + 1); + rc = strncpy_s(strs, strlen(interconn_url) + 1, interconn_url, strlen(interconn_url)); + securec_check_c(rc, "\0", "\0"); + + token = strtok_s(strs, delim, &next_token); + do { + nodeNum++; + token = strtok_s(NULL, delim, &next_token); + } while (token != NULL); + pfree(strs); + return nodeNum; +} + + /* * read ss config, return enable_dss * we will get ss_enable_dss, ss_dss_conn_path and ss_dss_vg_name. @@ -7458,6 +7485,7 @@ bool ss_read_config(void) { char config_file[MAXPGPATH] = {0}; char enable_dss[MAXPGPATH] = {0}; + char interconnect_url[MAXPGPATH] = {0}; char** optlines = NULL; int ret = EOK; @@ -7480,6 +7508,9 @@ bool ss_read_config(void) ss_instance_config.dss.vgname = (char*)malloc(sizeof(char) * MAXPGPATH); (void)find_guc_optval((const char**)optlines, "ss_dss_conn_path", ss_instance_config.dss.socketpath); (void)find_guc_optval((const char**)optlines, "ss_dss_vg_name", ss_instance_config.dss.vgname); + (void)find_guc_optval((const char**)optlines, "ss_interconnect_url", interconnect_url); + ss_instance_config.dss.interNodeNum = ss_get_inter_node_nums(interconnect_url); + freefile(optlines); optlines = NULL; return true; diff --git a/src/common/port/tool_common.cpp b/src/common/port/tool_common.cpp index ffee72a16..04adab881 100644 --- a/src/common/port/tool_common.cpp +++ b/src/common/port/tool_common.cpp @@ -34,6 +34,7 @@ SSInstanceConfig ss_instance_config = { .vglog = NULL, .vgdata = NULL, .socketpath = NULL, + .interNodeNum = 0, }, }; diff --git a/src/include/tool_common.h b/src/include/tool_common.h index b6a3bd36d..4f0b57eb5 100644 --- a/src/include/tool_common.h +++ b/src/include/tool_common.h @@ -115,6 +115,7 @@ typedef struct DssOptions { char *vglog; char *vgdata; char *socketpath; + int interNodeNum; } DssOptions; typedef struct SSInstanceConfig { From 46981778190a08ee707e154689e58a73a36d049b Mon Sep 17 00:00:00 2001 From: kenxx Date: Tue, 22 Aug 2023 06:05:50 -0400 Subject: [PATCH 151/304] fix_rename_tag --- src/gausskernel/process/tcop/utility.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/gausskernel/process/tcop/utility.cpp b/src/gausskernel/process/tcop/utility.cpp index 05d22c428..d13f0c8e5 100755 --- a/src/gausskernel/process/tcop/utility.cpp +++ b/src/gausskernel/process/tcop/utility.cpp @@ -8876,9 +8876,11 @@ const char* CreateCommandTag(Node* parse_tree) tag = "COPY"; break; - case T_RenameStmt: - tag = AlterObjectTypeCommandTag(((RenameStmt*)parse_tree)->renameType); + case T_RenameStmt: { + ObjectType RenameType = ((RenameStmt*)parse_tree)->renameType == OBJECT_COLUMN ? ((RenameStmt*)parse_tree)->relationType:((RenameStmt*)parse_tree)->renameType; + tag = AlterObjectTypeCommandTag(RenameType); break; + } case T_AlterObjectSchemaStmt: tag = AlterObjectTypeCommandTag(((AlterObjectSchemaStmt*)parse_tree)->objectType); From 6153c589ffff411102b7648acbba75637c224f2a Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Tue, 22 Aug 2023 19:12:27 +0800 Subject: [PATCH 152/304] =?UTF-8?q?=E6=97=A5=E5=BF=97=E5=90=88=E4=B8=80?= =?UTF-8?q?=E8=A1=A5=E5=85=853?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/pg_ctl/pg_ctl.cpp | 13 +- src/bin/pg_rewind/parsexlog.cpp | 37 +++- src/bin/pg_rewind/pg_rewind.cpp | 26 ++- src/common/port/tool_common.cpp | 4 +- .../ddes/adapter/ss_dms_bufmgr.cpp | 2 +- .../ddes/adapter/ss_dms_callback.cpp | 21 +- .../ddes/adapter/ss_dms_recovery.cpp | 4 +- .../ddes/adapter/ss_reform_common.cpp | 39 +++- .../process/postmaster/checkpointer.cpp | 2 +- .../process/postmaster/postmaster.cpp | 42 ++-- .../process/postmaster/startup.cpp | 4 +- .../storage/access/transam/xlog.cpp | 180 +++++++----------- src/gausskernel/storage/file/fd.cpp | 2 +- src/gausskernel/storage/ipc/procarray.cpp | 2 +- .../storage/replication/basebackup.cpp | 3 +- .../storage/replication/dataqueue.cpp | 2 +- .../storage/replication/datasyncrep.cpp | 2 +- .../storage/replication/libpqwalreceiver.cpp | 14 +- .../shared_storage_walreceiver.cpp | 6 +- .../replication/ss_cluster_replication.cpp | 6 +- .../storage/replication/syncrep.cpp | 5 +- .../storage/replication/walreceiver.cpp | 22 +-- .../storage/replication/walreceiverfuncs.cpp | 6 +- .../storage/replication/walsender.cpp | 20 +- src/gausskernel/storage/smgr/segstore.cpp | 4 +- .../xlog_share_storage/xlog_share_storage.cpp | 11 +- src/include/access/xlog.h | 8 - src/include/ddes/dms/ss_common_attr.h | 51 ----- src/include/ddes/dms/ss_dms_recovery.h | 1 + src/include/ddes/dms/ss_reform_common.h | 1 + .../replication/ss_cluster_replication.h | 24 +-- src/include/tool_common.h | 2 +- 32 files changed, 246 insertions(+), 320 deletions(-) diff --git a/src/bin/pg_ctl/pg_ctl.cpp b/src/bin/pg_ctl/pg_ctl.cpp index 4f23de9b9..67fcd8a55 100755 --- a/src/bin/pg_ctl/pg_ctl.cpp +++ b/src/bin/pg_ctl/pg_ctl.cpp @@ -6815,15 +6815,6 @@ int main(int argc, char** argv) } break; } - case 'g': - check_input_for_security(optarg); - if (atoi(optarg) < MIN_INSTANCEID || atoi(optarg) > MAX_INSTANCEID) { - pg_log(PG_WARNING, _("unexpected node id specified, valid range is %d - %d.\n"), - MIN_INSTANCEID, MAX_INSTANCEID ); - goto Error; - } - ss_instance_config.dss.instance_id = atoi(optarg); - break; case 1: clear_backup_dir = true; break; @@ -7485,6 +7476,7 @@ bool ss_read_config(void) { char config_file[MAXPGPATH] = {0}; char enable_dss[MAXPGPATH] = {0}; + char inst_id[MAXPGPATH] = {0}; char interconnect_url[MAXPGPATH] = {0}; char** optlines = NULL; int ret = EOK; @@ -7508,6 +7500,9 @@ bool ss_read_config(void) ss_instance_config.dss.vgname = (char*)malloc(sizeof(char) * MAXPGPATH); (void)find_guc_optval((const char**)optlines, "ss_dss_conn_path", ss_instance_config.dss.socketpath); (void)find_guc_optval((const char**)optlines, "ss_dss_vg_name", ss_instance_config.dss.vgname); + (void)find_guc_optval((const char**)optlines, "ss_instance_id", inst_id); + ss_instance_config.dss.instance_id = atoi(inst_id); + (void)find_guc_optval((const char**)optlines, "ss_interconnect_url", interconnect_url); ss_instance_config.dss.interNodeNum = ss_get_inter_node_nums(interconnect_url); diff --git a/src/bin/pg_rewind/parsexlog.cpp b/src/bin/pg_rewind/parsexlog.cpp index 3c7ca5681..62ee0e870 100644 --- a/src/bin/pg_rewind/parsexlog.cpp +++ b/src/bin/pg_rewind/parsexlog.cpp @@ -142,8 +142,7 @@ BuildErrorCode findCommonCheckpoint(const char* datadir, TimeLineID tli, XLogRec #endif XLogRecPtr max_lsn; char returnmsg[MAX_ERR_MSG_LENTH] = {0}; - char dssdirdata[MAXPGPATH] = {0}; - char* dssdir = dssdirdata; + char dssxlogdir[MAXPGPATH] = {0}; pg_crc32 maxLsnCrc = 0; XLogRecord* record = NULL; XLogRecPtr searchptr; @@ -156,17 +155,17 @@ BuildErrorCode findCommonCheckpoint(const char* datadir, TimeLineID tli, XLogRec TimestampTz start_time; TimestampTz current_time; + /* + * local max lsn must be exists, or change to full build. + */ if (ss_instance_config.dss.enable_dss) { - ret = snprintf_s(dssdirdata, MAXPGPATH, MAXPGPATH - 1, "%s/%s%d", ss_instance_config.dss.vgname, XLOGDIR, ss_instance_config.dss.instance_id); + ret = snprintf_s(dssxlogdir, MAXPGPATH, MAXPGPATH - 1, "%s/%s%d", + ss_instance_config.dss.vgname, XLOGDIR, ss_instance_config.dss.instance_id); securec_check_ss_c(ret, "", ""); + max_lsn = FindMaxLSN(datadir_target, returnmsg, XLOG_READER_MAX_MSGLENTH, &maxLsnCrc, NULL, NULL, dssxlogdir); } else { - dssdir = NULL; + max_lsn = FindMaxLSN(datadir_target, returnmsg, XLOG_READER_MAX_MSGLENTH, &maxLsnCrc); } - - /* - * local max lsn must be exists, or change to full build. - */ - max_lsn = FindMaxLSN(datadir_target, returnmsg, XLOG_READER_MAX_MSGLENTH, &maxLsnCrc, NULL, NULL, dssdir); if (XLogRecPtrIsInvalid(max_lsn)) { pg_fatal("find max lsn fail, errmsg:%s\n", returnmsg); return BUILD_FATAL; @@ -200,7 +199,25 @@ BuildErrorCode findCommonCheckpoint(const char* datadir, TimeLineID tli, XLogRec } uint8 info; - record = XLogReadRecord(xlogreader, searchptr, &errormsg, true, dssdir); + if (ss_instance_config.dss.enable_dss) { + struct dirent *entry; + DIR* dssdir = opendir(ss_instance_config.dss.vgname); + while (dssdir != NULL && (entry = readdir(dssdir)) != NULL) { + if (strncmp(entry->d_name, "pg_xlog", strlen("pg_xlog")) == 0) { + ret = snprintf_s(dssxlogdir, MAXPGPATH, MAXPGPATH - 1, "%s/%s", ss_instance_config.dss.vgname, entry->d_name); + securec_check_ss_c(ret, "", ""); + record = XLogReadRecord(xlogreader, searchptr, &errormsg, true, dssxlogdir); + if (record != NULL) { + break; + } + } else { + continue; + } + } + closedir(dssdir); + } else { + record = XLogReadRecord(xlogreader, searchptr, &errormsg); + } if (record == NULL) { if (errormsg != NULL) { pg_fatal("could not find previous WAL record at %X/%X: %s\n", diff --git a/src/bin/pg_rewind/pg_rewind.cpp b/src/bin/pg_rewind/pg_rewind.cpp index 3ee3318ca..bcac9409c 100755 --- a/src/bin/pg_rewind/pg_rewind.cpp +++ b/src/bin/pg_rewind/pg_rewind.cpp @@ -943,18 +943,13 @@ BuildErrorCode do_build_check(const char* pgdata, const char* connstr, char* sys XLogRecPtr startrec; errno_t errorno = EOK; BuildErrorCode rv = BUILD_SUCCESS; + char controlFile[MAXPGPATH]; - datadir_target = pg_strdup(pgdata); if (connstr_source == NULL) { connstr_source = pg_strdup(connstr); } - if (connstr_source == NULL) { - pg_log(PG_WARNING, "%s: no source specified (--source-server)\n", progname); - pg_log(PG_WARNING, "Try \"%s --help\" for more information.\n", progname); - return BUILD_ERROR; - } - + datadir_target = pg_strdup(pgdata); if (datadir_target == NULL) { pg_log(PG_WARNING, "%s: no target data directory specified (--target-pgdata)\n", progname); pg_log(PG_WARNING, "Try \"%s --help\" for more information.\n", progname); @@ -999,9 +994,14 @@ BuildErrorCode do_build_check(const char* pgdata, const char* connstr, char* sys * Ok, we have all the options and we're ready to start. Read in all the * information we need from both clusters. */ - buffer = slurpFile(ss_instance_config.dss.vgname, "pg_control", &size); + if (ss_instance_config.dss.enable_dss) { + buffer = slurpFile(ss_instance_config.dss.vgname, "pg_control", &size); + } else { + buffer = slurpFile(pgdata, "global/pg_control", &size); + } PG_CHECKBUILD_AND_RETURN(); - digestControlFile(&ControlFile_target, (const char*)buffer); + /* in share storage mode, all nodes's pg_control is in one file, we need offset BLCKSZ * id */ + digestControlFile(&ControlFile_target, (const char*)(buffer + BLCKSZ * ss_instance_config.dss.instance_id)); pg_free(buffer); buffer = NULL; PG_CHECKBUILD_AND_RETURN(); @@ -1013,7 +1013,13 @@ BuildErrorCode do_build_check(const char* pgdata, const char* connstr, char* sys (uint32)(ControlFile_target.checkPointCopy.redo >> 32), (uint32)(ControlFile_target.checkPointCopy.redo)); - buffer = fetchFile("+data/pg_control", &size); + if (ss_instance_config.dss.enable_dss) { + errorno = snprintf_s(controlFile, MAXPGPATH, MAXPGPATH - 1, "%s/pg_control", ss_instance_config.dss.vgname); + securec_check_ss_c(errorno, "\0", "\0"); + buffer = fetchFile(controlFile, &size); + } else { + buffer = fetchFile("global/pg_control", &size); + } PG_CHECKBUILD_AND_RETURN(); digestControlFile(&ControlFile_source, buffer); pg_free(buffer); diff --git a/src/common/port/tool_common.cpp b/src/common/port/tool_common.cpp index 04adab881..e118e6d61 100644 --- a/src/common/port/tool_common.cpp +++ b/src/common/port/tool_common.cpp @@ -28,13 +28,13 @@ SSInstanceConfig ss_instance_config = { .dss = { .enable_dss = false, - .instance_id = -1, + .instance_id = 0, .primaryInstId = -1, + .interNodeNum = 0, .vgname = NULL, .vglog = NULL, .vgdata = NULL, .socketpath = NULL, - .interNodeNum = 0, }, }; diff --git a/src/gausskernel/ddes/adapter/ss_dms_bufmgr.cpp b/src/gausskernel/ddes/adapter/ss_dms_bufmgr.cpp index 917f63303..9786f9f6d 100644 --- a/src/gausskernel/ddes/adapter/ss_dms_bufmgr.cpp +++ b/src/gausskernel/ddes/adapter/ss_dms_bufmgr.cpp @@ -742,7 +742,7 @@ dms_session_e DMSGetProcType4RequestPage() * DMS_SESSION_RECOVER_HOT_STANDBY will be returned, it indicates that normal threads can access * page in recovery state. */ - if ((SS_STANDBY_CLUSTER_MAIN_STANDBY || IS_SS_REPLICATION_MAIN_STANBY_NODE) && pmState == PM_HOT_STANDBY) { + if (SS_REPLICATION_MAIN_STANBY_NODE && pmState == PM_HOT_STANDBY) { return DMS_SESSION_RECOVER_HOT_STANDBY; } else { return DMS_SESSION_RECOVER; diff --git a/src/gausskernel/ddes/adapter/ss_dms_callback.cpp b/src/gausskernel/ddes/adapter/ss_dms_callback.cpp index 24cf0ffd5..05f160755 100644 --- a/src/gausskernel/ddes/adapter/ss_dms_callback.cpp +++ b/src/gausskernel/ddes/adapter/ss_dms_callback.cpp @@ -58,7 +58,7 @@ void SSWakeupRecovery(void) /* need make sure pagewriter started first */ bool need_recovery = true; - if (DORADO_STANDBY_CLUSTER || SS_REPLICATION_STANDBY_CLUSTER) { + if (SS_REPLICATION_STANDBY_CLUSTER) { g_instance.dms_cxt.SSRecoveryInfo.recovery_pause_flag = false; return; } @@ -200,9 +200,8 @@ static int CBGetTxnCSN(void *db_handle, dms_opengauss_xid_csn_t *csn_req, dms_op static int CBGetSnapshotData(void *db_handle, dms_opengauss_txn_snapshot_t *txn_snapshot, uint8 inst_id) { - /* SS_STANDBY_CLUSTER_NORMAL_MAIN_STANDBY always is in recovery progress, but it can acquire snapshot*/ - if (RecoveryInProgress() && - !(SS_STANDBY_CLUSTER_NORMAL_MAIN_STANDBY || (SS_NORMAL_PRIMARY && IS_SS_REPLICATION_MAIN_STANBY_NODE))) { + /* SS_REPLICATION_MAIN_STANBY_NODE always is in recovery progress, but it can acquire snapshot*/ + if (RecoveryInProgress() && !(SS_NORMAL_PRIMARY && SS_REPLICATION_MAIN_STANBY_NODE)) { return DMS_ERROR; } @@ -421,7 +420,7 @@ static void CBSwitchoverResult(void *db_handle, int result) } else { /* abort and restore state */ g_instance.dms_cxt.SSClusterState = NODESTATE_NORMAL; - if (DORADO_STANDBY_CLUSTER || SS_REPLICATION_STANDBY_CLUSTER) { + if (SS_REPLICATION_STANDBY_CLUSTER) { g_instance.dms_cxt.SSReformInfo.in_reform = false; } ereport(WARNING, (errmodule(MOD_DMS), errmsg("[SS switchover] Switchover failed, errno: %d.", result))); @@ -1582,7 +1581,7 @@ static void CBReformSetDmsRole(void *db_handle, unsigned int reformer_id) if (new_dms_role == DMS_ROLE_REFORMER) { ereport(LOG, (errmodule(MOD_DMS), errmsg("[SS switchover]begin to set currrent DSS as primary"))); /* standby of standby cluster need to set mode to STANDBY_MODE in dual cluster*/ - if (DORADO_STANDBY_CLUSTER || SS_REPLICATION_STANDBY_CLUSTER) { + if (SS_REPLICATION_STANDBY_CLUSTER) { t_thrd.postmaster_cxt.HaShmData->current_mode = STANDBY_MODE; } while (dss_set_server_status_wrapper() != GS_SUCCESS) { @@ -1757,8 +1756,8 @@ static void CBReformStartNotify(void *db_handle, dms_role_t role, unsigned char } /* After reform done, standby of standby cluster need to set mode to STANDBY_MODE in dual cluster. */ - if (SS_REFORM_REFORMER && (g_instance.attr.attr_common.cluster_run_mode == RUN_MODE_STANDBY) && - (g_instance.attr.attr_storage.xlog_file_path != NULL || SS_CLUSTER_DORADO_REPLICATION)) { + if (SS_REFORM_REFORMER && SS_REPLICATION_DORADO_CLUSTER && + g_instance.attr.attr_common.cluster_run_mode == RUN_MODE_STANDBY) { t_thrd.postmaster_cxt.HaShmData->current_mode = STANDBY_MODE; } } @@ -1774,8 +1773,7 @@ static int CBReformDoneNotify(void *db_handle) } /* After reform done, primary of master cluster need to set mode to PRIMARY_MODE in dual cluster. */ - if (SS_REFORM_REFORMER && (g_instance.attr.attr_common.cluster_run_mode == RUN_MODE_PRIMARY) && - (g_instance.attr.attr_storage.xlog_file_path != NULL || SS_CLUSTER_DORADO_REPLICATION)) { + if (SS_REFORM_REFORMER && SS_REPLICATION_PRIMARY_CLUSTER) { t_thrd.postmaster_cxt.HaShmData->current_mode = PRIMARY_MODE; } @@ -1857,8 +1855,7 @@ static int CBUpdateNodeOldestXmin(void *db_handle, uint8 inst_id, unsigned long void DmsCallbackThreadShmemInit(unsigned char need_startup, char **reg_data) { /* in dorado mode, we need to wait sharestorageinit finished */ - while (!g_instance.dms_cxt.SSRecoveryInfo.dorado_sharestorage_inited && - (g_instance.attr.attr_storage.xlog_file_path != NULL || SS_CLUSTER_DORADO_REPLICATION)) { + while (!g_instance.dms_cxt.SSRecoveryInfo.dorado_sharestorage_inited && SS_REPLICATION_DORADO_CLUSTER) { pg_usleep(REFORM_WAIT_TIME); } IsUnderPostmaster = true; diff --git a/src/gausskernel/ddes/adapter/ss_dms_recovery.cpp b/src/gausskernel/ddes/adapter/ss_dms_recovery.cpp index 92c38eb6d..f64bc60db 100644 --- a/src/gausskernel/ddes/adapter/ss_dms_recovery.cpp +++ b/src/gausskernel/ddes/adapter/ss_dms_recovery.cpp @@ -126,7 +126,7 @@ bool SSRecoveryNodes() * recovery phase could be regarded successful in hot_standby thus set pmState = PM_HOT_STANDBY, which * indicate database systerm is ready to accept read only connections. */ - if ((SS_STANDBY_CLUSTER_MAIN_STANDBY || IS_SS_REPLICATION_MAIN_STANBY_NODE) && pmState == PM_HOT_STANDBY) { + if (SS_REPLICATION_MAIN_STANBY_NODE && pmState == PM_HOT_STANDBY) { result = true; break; } @@ -147,7 +147,7 @@ bool SSRecoveryApplyDelay() return false; } - if (DORADO_STANDBY_CLUSTER || SS_REPLICATION_STANDBY_CLUSTER) { + if (SS_REPLICATION_STANDBY_CLUSTER) { return true; } diff --git a/src/gausskernel/ddes/adapter/ss_reform_common.cpp b/src/gausskernel/ddes/adapter/ss_reform_common.cpp index 25ab24c4e..38d411a29 100644 --- a/src/gausskernel/ddes/adapter/ss_reform_common.cpp +++ b/src/gausskernel/ddes/adapter/ss_reform_common.cpp @@ -101,10 +101,10 @@ int SSXLogFileOpenAnyTLI(XLogSegNo segno, int emode, uint32 sources, char* xlog_ } /* - * When SS_CLUSTER_DORADO_REPLICATION enabled, current xlog dictionary may be not the correct dictionary, + * When SS_REPLICATION_DORADO_CLUSTER enabled, current xlog dictionary may be not the correct dictionary, * because all xlog dictionaries are in the same LUN, we need loop over other dictionaries. */ - if (fd < 0 && SS_CLUSTER_DORADO_REPLICATION) { + if (fd < 0 && SS_REPLICATION_DORADO_CLUSTER) { return -1; } @@ -144,7 +144,7 @@ int SSReadXlogInternal(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, XL * record just writing into pg_xlog file when source is XLOG_FROM_STREAM and dms and dss are enabled. So we * need to reread xlog from dss to preReadBuf. */ - if (SS_STANDBY_CLUSTER_MAIN_STANDBY) { + if (SS_REPLICATION_MAIN_STANBY_NODE) { volatile XLogCtlData *xlogctl = t_thrd.shemem_ptr_cxt.XLogCtl; if (XLByteInPreReadBuf(targetPagePtr, xlogreader->preReadStartPtr) && ((targetRecPtr < xlogFlushPtrForPerRead && t_thrd.xlog_cxt.readSource == XLOG_FROM_STREAM) || @@ -154,13 +154,13 @@ int SSReadXlogInternal(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, XL } if ((XLByteInPreReadBuf(targetPagePtr, xlogreader->preReadStartPtr) && - !SS_STANDBY_CLUSTER_MAIN_STANDBY) || (!isReadFile)) { + !SS_REPLICATION_MAIN_STANBY_NODE) || (!isReadFile)) { preReadOff = targetPagePtr % XLogPreReadSize; int err = memcpy_s(buf, readLen, xlogreader->preReadBuf + preReadOff, readLen); securec_check(err, "\0", "\0"); break; } else { - if (SS_STANDBY_CLUSTER_MAIN_STANDBY) { + if (SS_REPLICATION_MAIN_STANBY_NODE) { xlogreader->xlogFlushPtrForPerRead = GetWalRcvWriteRecPtr(NULL); xlogFlushPtrForPerRead = xlogreader->xlogFlushPtrForPerRead; } @@ -220,6 +220,35 @@ void SSGetRecoveryXlogPath() securec_check_ss(rc, "", ""); } +void SSDoradoGetInstidList() +{ + for (int i = 0; i < DMS_MAX_INSTANCE; i++) { + g_instance.dms_cxt.SSRecoveryInfo.instid_list[i] = -1; + } + struct dirent *entry; + errno_t rc = EOK; + DIR* dssdir = opendir(g_instance.attr.attr_storage.dss_attr.ss_dss_vg_name); + if (dssdir == NULL) { + ereport(PANIC, (errcode_for_file_access(), errmsg("Error opening dssdir %s", + g_instance.attr.attr_storage.dss_attr.ss_dss_vg_name))); + } + + int len = strlen("pg_xlog"); + int index = 0; + while ((entry = readdir(dssdir)) != NULL) { + if (strncmp(entry->d_name, "pg_xlog", len) == 0) { + if (strlen(entry->d_name) > len) { + rc = memmove_s(entry->d_name, MAX_PATH, entry->d_name + len, strlen(entry->d_name) - len + 1); + securec_check_c(rc, "\0", "\0"); + g_instance.dms_cxt.SSRecoveryInfo.instid_list[index++] = atoi(entry->d_name); + } + } else { + continue; + } + } + closedir(dssdir); +} + static void SSSaveOldReformerCtrl() { ss_reformer_ctrl_t new_ctrl = g_instance.dms_cxt.SSReformerControl; diff --git a/src/gausskernel/process/postmaster/checkpointer.cpp b/src/gausskernel/process/postmaster/checkpointer.cpp index 527f81c23..3ddaa5a28 100755 --- a/src/gausskernel/process/postmaster/checkpointer.cpp +++ b/src/gausskernel/process/postmaster/checkpointer.cpp @@ -543,7 +543,7 @@ void CheckpointerMain(void) } else { CheckPointBuffers(flags, true); } - } else if (!do_restartpoint && !(DORADO_STANDBY_CLUSTER_MAINSTANDBY_NODE || IS_SS_REPLICATION_MAIN_STANBY_NODE)) { + } else if (!do_restartpoint && !SS_REPLICATION_MAIN_STANBY_NODE) { CreateCheckPoint(flags); ckpt_performed = true; if (!bgwriter_first_startup && CheckFpwBeforeFirstCkpt()) { diff --git a/src/gausskernel/process/postmaster/postmaster.cpp b/src/gausskernel/process/postmaster/postmaster.cpp index f2e46209c..ea5d789a7 100644 --- a/src/gausskernel/process/postmaster/postmaster.cpp +++ b/src/gausskernel/process/postmaster/postmaster.cpp @@ -3708,7 +3708,7 @@ void ArchObsThreadManage() static bool IsNeedStartXlogCopyer() { /* Only in normal cluster replication and ss cluster replication, we need start xlogcopyer thread. */ - if (SS_CLUSTER_DORADO_REPLICATION || !IS_SHARED_STORAGE_MODE) { + if (!IS_SHARED_STORAGE_MODE) { return false; } @@ -3781,7 +3781,7 @@ static int ServerLoop(void) fd_set rmask; int selres; - if (t_thrd.postmaster_cxt.HaShmData->current_mode != NORMAL_MODE || IS_SHARED_STORAGE_MODE || SS_CLUSTER_DORADO_REPLICATION) { + if (t_thrd.postmaster_cxt.HaShmData->current_mode != NORMAL_MODE || IS_SHARED_STORAGE_MODE || SS_REPLICATION_DORADO_CLUSTER) { check_and_reset_ha_listen_port(); #ifdef HAVE_POLL @@ -6642,7 +6642,7 @@ static void ProcessDemoteRequest(void) signal_child(g_instance.pid_cxt.StatementPID, SIGTERM); } - if (g_instance.pid_cxt.StartupPID != 0 && (DORADO_STANDBY_CLUSTER || SS_REPLICATION_STANDBY_CLUSTER)) { + if (g_instance.pid_cxt.StartupPID != 0 && (SS_REPLICATION_STANDBY_CLUSTER)) { signal_child(g_instance.pid_cxt.StartupPID, SIGTERM); } @@ -6989,8 +6989,7 @@ static void reaper(SIGNAL_ARGS) } if (g_instance.pid_cxt.sharedStorageXlogCopyThreadPID == 0 && !dummyStandbyMode && - g_instance.attr.attr_storage.xlog_file_path != NULL && (!SS_PRIMARY_STANDBY_CLUSTER_STANDBY - && !SS_PRIMARY_DEMOTING)) { + g_instance.attr.attr_storage.xlog_file_path != NULL) { g_instance.pid_cxt.sharedStorageXlogCopyThreadPID = initialize_util_thread(SHARE_STORAGE_XLOG_COPYER); } @@ -9979,10 +9978,10 @@ static void sigusr1_handler(SIGNAL_ARGS) * update cluster_run_mode from pg_control file, * in case failover has been performed between two dorado cluster. */ - if (ENABLE_DMS && g_instance.attr.attr_storage.xlog_file_path != 0) { + if (SS_REPLICATION_DORADO_CLUSTER) { g_instance.attr.attr_common.cluster_run_mode = g_instance.dms_cxt.SSReformerControl.clusterRunMode; } - if (ENABLE_DMS && DORADO_STANDBY_CLUSTER_MAINSTANDBY_NODE) { + if (SS_REPLICATION_MAIN_STANBY_NODE) { ereport(LOG, (errmsg("Failover between two dorado cluster start, change current run mode to primary_cluster"))); g_instance.attr.attr_common.cluster_run_mode = RUN_MODE_PRIMARY; @@ -10112,9 +10111,9 @@ static void sigusr1_handler(SIGNAL_ARGS) if (CheckPostmasterSignal(PMSIGNAL_START_WALRECEIVER) && g_instance.pid_cxt.WalReceiverPID == 0 && (pmState == PM_STARTUP || pmState == PM_RECOVERY || pmState == PM_HOT_STANDBY || pmState == PM_WAIT_READONLY) && g_instance.status == NoShutdown && - (!ENABLE_DMS || SS_STANDBY_CLUSTER_MAIN_STANDBY || SS_CLUSTER_DORADO_REPLICATION)) { - /* when SS_CLUSTER_DORADO_REPLICATION enabled, don't start walrecwrite */ - if (g_instance.pid_cxt.WalRcvWriterPID == 0 && !SS_CLUSTER_DORADO_REPLICATION) { + (!ENABLE_DMS || SS_REPLICATION_DORADO_CLUSTER)) { + /* when SS_REPLICATION_DORADO_CLUSTER enabled, don't start walrecwrite */ + if (g_instance.pid_cxt.WalRcvWriterPID == 0 && !SS_REPLICATION_DORADO_CLUSTER) { g_instance.pid_cxt.WalRcvWriterPID = initialize_util_thread(WALRECWRITE); SetWalRcvWriterPID(g_instance.pid_cxt.WalRcvWriterPID); } @@ -10247,23 +10246,23 @@ static void sigusr1_handler(SIGNAL_ARGS) signal_child(g_instance.pid_cxt.WLMCollectPID, SIGTERM); } - if (g_instance.pid_cxt.UndoLauncherPID != 0 && (DORADO_STANDBY_CLUSTER ||SS_REPLICATION_STANDBY_CLUSTER)) { + if (g_instance.pid_cxt.UndoLauncherPID != 0 && (SS_REPLICATION_STANDBY_CLUSTER)) { signal_child(g_instance.pid_cxt.UndoLauncherPID, SIGTERM); } #ifndef ENABLE_MULTIPLE_NODES - if (g_instance.pid_cxt.ApplyLauncerPID != 0 && (DORADO_STANDBY_CLUSTER ||SS_REPLICATION_STANDBY_CLUSTER)) { + if (g_instance.pid_cxt.ApplyLauncerPID != 0 && (SS_REPLICATION_STANDBY_CLUSTER)) { signal_child(g_instance.pid_cxt.ApplyLauncerPID, SIGTERM); } #endif - if (g_instance.pid_cxt.GlobalStatsPID != 0 && (DORADO_STANDBY_CLUSTER ||SS_REPLICATION_STANDBY_CLUSTER)) { + if (g_instance.pid_cxt.GlobalStatsPID != 0 && (SS_REPLICATION_STANDBY_CLUSTER)) { signal_child(g_instance.pid_cxt.GlobalStatsPID, SIGTERM); } - if (g_instance.pid_cxt.UndoRecyclerPID != 0 && (DORADO_STANDBY_CLUSTER ||SS_REPLICATION_STANDBY_CLUSTER)) { + if (g_instance.pid_cxt.UndoRecyclerPID != 0 && (SS_REPLICATION_STANDBY_CLUSTER)) { signal_child(g_instance.pid_cxt.UndoRecyclerPID, SIGTERM); } - if (g_instance.pid_cxt.FaultMonitorPID != 0 && (DORADO_STANDBY_CLUSTER ||SS_REPLICATION_STANDBY_CLUSTER)) { + if (g_instance.pid_cxt.FaultMonitorPID != 0 && (SS_REPLICATION_STANDBY_CLUSTER)) { signal_child(g_instance.pid_cxt.FaultMonitorPID, SIGTERM); } @@ -12700,16 +12699,11 @@ const char* wal_get_db_state_string(DbState db_state) static ServerMode get_cur_mode(void) { - if (ENABLE_DMS) { - if (DORADO_STANDBY_CLUSTER || SS_REPLICATION_STANDBY_CLUSTER) { - return STANDBY_MODE; - } - /* except for main standby in standby cluster, current mode of instance is determined by SS_OFFICIAL_PRIMARY*/ - if (g_instance.attr.attr_storage.xlog_file_path !=0 && SS_OFFICIAL_PRIMARY && - t_thrd.postmaster_cxt.HaShmData->current_mode == STANDBY_MODE) { + if (SS_REPLICATION_DORADO_CLUSTER) { + if (SS_REPLICATION_MAIN_STANBY_NODE) { return STANDBY_MODE; } - return !SS_OFFICIAL_PRIMARY ? STANDBY_MODE : PRIMARY_MODE; + return SS_OFFICIAL_PRIMARY ? PRIMARY_MODE : STANDBY_MODE; } return t_thrd.postmaster_cxt.HaShmData->current_mode; } @@ -14973,7 +14967,7 @@ void InitShmemForDmsCallBack() const char *GetSSServerMode(ServerMode mode) { - if (IS_SHARED_STORAGE_MODE || SS_CLUSTER_DORADO_REPLICATION) { + if (IS_SHARED_STORAGE_MODE || SS_REPLICATION_DORADO_CLUSTER) { if (SS_OFFICIAL_PRIMARY && (mode == PRIMARY_MODE || mode == NORMAL_MODE)) { return "Primary"; } diff --git a/src/gausskernel/process/postmaster/startup.cpp b/src/gausskernel/process/postmaster/startup.cpp index 2a47f3027..65d06d7be 100755 --- a/src/gausskernel/process/postmaster/startup.cpp +++ b/src/gausskernel/process/postmaster/startup.cpp @@ -232,7 +232,7 @@ void HandleStartupProcInterrupts(void) * Check if we were requested to exit without finishing recovery. */ if (t_thrd.startup_cxt.shutdown_requested && SmartShutdown != g_instance.status) { - if (t_thrd.xlog_cxt.StandbyModeRequested && IS_SHARED_STORAGE_STANDBY_CLUSTER && ENABLE_DMS) { + if (t_thrd.xlog_cxt.StandbyModeRequested && SS_REPLICATION_MAIN_STANBY_NODE) { ereport(LOG, (errmsg("dorado standby cluster switchover shutdown startup\n"))); if (!IsExtremeRedo()) { DisownLatch(&t_thrd.shemem_ptr_cxt.XLogCtl->recoveryWakeupLatch); @@ -285,7 +285,7 @@ static void StartupReleaseAllLocks(int code, Datum arg) void DeleteDisConnFileInClusterStandby() { - if (!(IS_SHARED_STORAGE_MODE || SS_CLUSTER_DORADO_REPLICATION)) { + if (!(IS_SHARED_STORAGE_MODE || SS_REPLICATION_DORADO_CLUSTER)) { return; } diff --git a/src/gausskernel/storage/access/transam/xlog.cpp b/src/gausskernel/storage/access/transam/xlog.cpp index ad90f9ce9..3a85240b5 100755 --- a/src/gausskernel/storage/access/transam/xlog.cpp +++ b/src/gausskernel/storage/access/transam/xlog.cpp @@ -4202,12 +4202,12 @@ static int XLogFileOpenInternal(XLogSegNo segno, const char *xlog_dir) (uint32)((segno) / XLogSegmentsPerXLogId), (uint32)((segno) % XLogSegmentsPerXLogId)); securec_check_ss(errorno, "", ""); - if (SS_CLUSTER_DORADO_REPLICATION) { + if (SS_REPLICATION_DORADO_CLUSTER) { fd = SSErgodicOpenXlogFile(segno, O_RDWR | PG_BINARY | (unsigned int)get_sync_bit(u_sess->attr.attr_storage.sync_method), - S_IRUSR | S_IWUSR); + S_IRUSR | S_IWUSR); } else { fd = BasicOpenFile(path, O_RDWR | PG_BINARY | (unsigned int)get_sync_bit(u_sess->attr.attr_storage.sync_method), - S_IRUSR | S_IWUSR); + S_IRUSR | S_IWUSR); } if (fd < 0) { @@ -4320,11 +4320,11 @@ static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, int source, } /* - * When SS_CLUSTER_DORADO_REPLICATION enabled, current xlog dictionary may be not the correct dictionary, + * When SS_REPLICATION_DORADO_CLUSTER enabled, current xlog dictionary may be not the correct dictionary, * because all xlog dictionaries are in the same LUN, we need loop over other dictionaries. * Do we need source == XLOG_FROM_STREAM? */ - if (SS_CLUSTER_DORADO_REPLICATION && source == XLOG_FROM_STREAM) { + if (SS_REPLICATION_DORADO_CLUSTER && source == XLOG_FROM_STREAM) { std::vector nodeList = SSGetAllStableNodeId(); // stable node list, Assert(!nodeList.empty()); char xlogPath[MAXPGPATH]; @@ -5219,6 +5219,39 @@ static void CleanupBackupHistory(void) FreeDir(xldir); } +/* +* in ss dorado double cluster, we need read xlogpath ergodic， +* we will read xlog in path where last read success +*/ +XLogRecord *SSXLogReadRecordErgodic(XLogReaderState *state, XLogRecPtr RecPtr, + char **errormsg, char* dssdata, int* idList) { + char xlogPath[MAXPGPATH]; + XLogRecord *record = NULL; + errno_t errorno = 0; + + for (int i = 0; i < DMS_MAX_INSTANCE; i++) { + if (idList[i] == -1) { + break; + } + errorno = snprintf_s(xlogPath, MAXPGPATH, MAXPGPATH - 1, "%s/%s%d", dssdata, "pg_xlog", idList[i]); + securec_check_ss(errorno, "", ""); + record = XLogReadRecord(state, RecPtr, errormsg, true, xlogPath); + if (record != NULL) { + /* read success, exchange index */ + int buf = idList[i]; + idList[i] = idList[0]; + idList[0] = buf; + break; + } else { + if (t_thrd.xlog_cxt.readFile >= 0) { + close(t_thrd.xlog_cxt.readFile); + t_thrd.xlog_cxt.readFile = -1; + } + } + } + return record; +} + /* * Attempt to read an XLOG record. * @@ -5248,30 +5281,9 @@ static XLogRecord *ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, in for (;;) { char *errormsg = NULL; - if (SS_CLUSTER_DORADO_REPLICATION) { - char xlog_path_ergodic[MAXPGPATH]; - struct dirent *entry; - errno_t errorno = EOK; - DIR* dssdir = opendir(g_instance.attr.attr_storage.dss_attr.ss_dss_vg_name); - if (dssdir == NULL) { - ereport(PANIC, (errcode_for_file_access(), errmsg("Error opening dssdir %s", - g_instance.attr.attr_storage.dss_attr.ss_dss_vg_name))); - } - - while ((entry = readdir(dssdir)) != NULL) { - if (strncmp(entry->d_name, "pg_xlog", strlen("pg_xlog")) == 0) { - errorno = snprintf_s(xlog_path_ergodic, MAXPGPATH, MAXPGPATH - 1, "%s/%s", - g_instance.attr.attr_storage.dss_attr.ss_dss_vg_name, entry->d_name); - securec_check_ss(errorno, "", ""); - record = XLogReadRecord(xlogreader, RecPtr, &errormsg, true, xlog_path_ergodic); - if (record != NULL) { - break; - } - } else { - continue; - } - } - closedir(dssdir); + if (SS_REPLICATION_DORADO_CLUSTER) { + record = SSXLogReadRecordErgodic(xlogreader, RecPtr, &errormsg, + g_instance.attr.attr_storage.dss_attr.ss_dss_vg_name, g_instance.dms_cxt.SSRecoveryInfo.instid_list); } else { record = XLogReadRecord(xlogreader, RecPtr, &errormsg); } @@ -6840,7 +6852,7 @@ void BootStrapXLOG(void) ctlInfo->version, ctlInfo->insertHead, ctlInfo->insertTail))); } - if (SS_CLUSTER_DORADO_REPLICATION) { + if (SS_REPLICATION_DORADO_CLUSTER) { ShareStorageXLogCtl *ctlInfo = g_instance.xlog_cxt.ssReplicationXLogCtl; InitSSDoradoCtlInfo(ctlInfo, sysidentifier); ctlInfo->insertHead = MAXALIGN((recptr - (char *)page)); @@ -9137,6 +9149,9 @@ void StartupXLOG(void) if (ENABLE_DMS && ENABLE_DSS) { SSGetRecoveryXlogPath(); + if (SS_REPLICATION_DORADO_CLUSTER) { + SSDoradoGetInstidList(); + } xlogreader = SSXLogReaderAllocate(&SSXLogPageRead, &readprivate, ALIGNOF_BUFFER); close_readFile_if_open(); if (SS_STANDBY_FAILOVER || SS_STANDBY_PROMOTING) { @@ -9614,7 +9629,7 @@ void StartupXLOG(void) } } - if (SS_PRIMARY_MODE || IS_SS_REPLICATION_MAIN_STANBY_NODE) { + if (SS_PRIMARY_MODE || SS_REPLICATION_MAIN_STANBY_NODE) { g_instance.dms_cxt.SSReformerControl.clusterRunMode = (ClusterRunMode)g_instance.attr.attr_common.cluster_run_mode; SSSaveReformerCtrl(); } @@ -10260,12 +10275,12 @@ void StartupXLOG(void) /* only primary mode can call getwritepermissionsharedstorage wnen dorado hyperreplication * and dms enabled. */ - if(!SS_PRIMARY_STANDBY_CLUSTER_STANDBY) { + if(IS_SHARED_STORAGE_MODE) { GetWritePermissionSharedStorage(); CheckShareStorageCtlInfo(EndOfLog); } - if (IS_SS_REPLICATION_PRIMARY_NODE) { + if (SS_REPLICATION_PRIMARY_NODE) { CheckSSDoradoCtlInfo(EndOfLog); } @@ -10515,30 +10530,6 @@ void StartupXLOG(void) /* Shut down readFile facility, free space. */ ShutdownReadFileFacility(); - /* When ss dorado enabled, and standby promoting, we don't need to copy */ - if (SS_STANDBY_FAILOVER && SS_PRIMARY_CLUSTER_STANDBY) { - ereport(LOG, (errmodule(MOD_DMS), - errmsg("[SS failover] standby promoting: copy endofxlog pageptr from primary to standby"))); - XLogRecPtr EndOfLogPagePtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - SSXLOGCopyFromOldPrimary(xlogreader, EndOfLogPagePtr); - - ereport(LOG, (errmodule(MOD_DMS), errmsg("[SS failover] standby promoting: copy xlog from local to dorado"))); - ShareStorageXLogCtl *sharestorageCtl = g_instance.xlog_cxt.shareStorageXLogCtl; - XLogRecPtr localFlush = InvalidXLogRecPtr; - localFlush = EndOfLog; - XLogRecPtr maxPosCanWrite = GetMaxPosCanOverWrite(); - uint32 shiftSize = 32; - ereport(LOG, - (errmsg("start to copy xlog from local to shared storage, localFlush: %X/%X, maxPosCanWrite: %X/%X.", - static_cast(localFlush >> shiftSize), static_cast(localFlush), - static_cast(maxPosCanWrite >> shiftSize), static_cast(maxPosCanWrite)))); - - if (XLByteLT(sharestorageCtl->insertHead, maxPosCanWrite)) { - XLogRecPtr expectPos = XLByteLT(localFlush, maxPosCanWrite) ? localFlush : maxPosCanWrite; - SSDoXLogCopyFromLocal(expectPos); - } - } - /* Shut down the xlog reader facility. */ XLogReaderFree(xlogreader); xlogreader = NULL; @@ -10701,8 +10692,7 @@ void StartupXLOG(void) } #endif - if (ENABLE_DMS && ENABLE_REFORM && !SS_PRIMARY_DEMOTED && - !(DORADO_STANDBY_CLUSTER || SS_REPLICATION_STANDBY_CLUSTER)) { + if (ENABLE_DMS && ENABLE_REFORM && !SS_PRIMARY_DEMOTED && !SS_REPLICATION_STANDBY_CLUSTER) { StartupWaitReform(); } } @@ -11422,7 +11412,7 @@ void ShutdownXLOG(int code, Datum arg) { if (SS_PRIMARY_DEMOTING) { ereport(LOG, (errmsg("[SS switchover] primary demote: doing shutdown checkpoint"))); - if (DORADO_STANDBY_CLUSTER || SS_REPLICATION_STANDBY_CLUSTER) { + if (SS_REPLICATION_STANDBY_CLUSTER) { CreateRestartPoint(CHECKPOINT_IS_SHUTDOWN | CHECKPOINT_IMMEDIATE); } else { CreateCheckPoint(CHECKPOINT_IS_SHUTDOWN | CHECKPOINT_IMMEDIATE); @@ -18868,7 +18858,7 @@ void UpdatePostgresqlFile(const char *optName, const char *gucLine) void SSWriteDoradoCtlInfoFile(int fd, char* buffer) { - Assert(ENABLE_DSS && SS_CLUSTER_DORADO_REPLICATION); + Assert(ENABLE_DSS && SS_REPLICATION_DORADO_CLUSTER); Assert(fd > 0); errno = EOK; if (pwrite(fd, buffer, SS_DORADO_CTL_INFO_SIZE, 0) != SS_DORADO_CTL_INFO_SIZE) { @@ -18885,7 +18875,7 @@ void SSWriteDoradoCtlInfoFile(int fd, char* buffer) void SSReadDoradoCtlInfoFile() { - Assert(ENABLE_DSS && SS_CLUSTER_DORADO_REPLICATION); + Assert(ENABLE_DSS && SS_REPLICATION_DORADO_CLUSTER); int fd = BasicOpenFile(SS_DORADO_CTRL_FILE, O_RDWR | PG_BINARY | O_DIRECT, S_IRUSR | S_IWUSR); if (fd < 0) { ereport(FATAL, (errcode_for_file_access(), errmsg("could not open ss ctl into file \"%s\": %m", SS_DORADO_CTRL_FILE))); @@ -18939,7 +18929,7 @@ void NormalClusterDoradoStorageInit() void ShareStorageInit() { - if (SS_CLUSTER_DORADO_REPLICATION) { + if (SS_REPLICATION_DORADO_CLUSTER) { SSClusterDoradoStorageInit(); } else if (IS_SHARED_STORAGE_MODE) { NormalClusterDoradoStorageInit(); @@ -19410,28 +19400,6 @@ static int SSDoradoReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePt CheckRecoveryConsistency(); if (WalRcvInProgress()) { bool havedata = false; - /* - * If we find an invalid record in the WAL streamed from - * master, something is seriously wrong. There's little - * chance that the problem will just go away, but PANIC is - * not good for availability either, especially in hot - * standby mode. Disconnect, and retry from - * archive/pg_xlog again. The WAL in the archive should be - * identical to what was streamed, so it's unlikely that - * it helps, but one can hope... - */ - if (t_thrd.xlog_cxt.failedSources & XLOG_FROM_STREAM) { - ProcTxnWorkLoad(true); - ereport(LOG, (errmsg("read from stream failed, request xlog receivedupto at %X/%X. targetRecPtr:%X/%x", - (uint32)(t_thrd.xlog_cxt.receivedUpto >> 32), - (uint32)t_thrd.xlog_cxt.receivedUpto, - (uint32)(targetRecPtr >> 32), - (uint32)targetRecPtr - ))); - ShutdownWalRcv(); - continue; - } - /* * Walreceiver is active, so see if new data has arrived. * @@ -19444,8 +19412,6 @@ static int SSDoradoReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePt * XLogReceiptTime will not advance, so the grace time * alloted to conflicting queries will decrease. */ - - if (XLByteLT(expectedRecPtr, t_thrd.xlog_cxt.receivedUpto)) { havedata = true; } else { @@ -19464,7 +19430,6 @@ static int SSDoradoReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePt } else { havedata = false; } - } if (havedata) { t_thrd.xlog_cxt.readSource = XLOG_FROM_STREAM; @@ -19475,7 +19440,7 @@ static int SSDoradoReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePt t_thrd.xlog_cxt.RedoDone = IsRedoDonePromoting(); pg_memory_barrier(); - if (IS_SS_REPLICATION_MAIN_STANBY_NODE && WalRcvIsDone() && CheckForFailoverTrigger()) { + if (SS_REPLICATION_MAIN_STANBY_NODE && WalRcvIsDone() && CheckForFailoverTrigger()) { t_thrd.xlog_cxt.receivedUpto = GetWalRcvWriteRecPtr(NULL); if (XLByteLT(RecPtr, t_thrd.xlog_cxt.receivedUpto)) { /* wait xlog redo done */ @@ -19526,7 +19491,7 @@ static int SSDoradoReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePt * existing file from pg_xlog. */ sources = XLOG_FROM_ARCHIVE | XLOG_FROM_PG_XLOG; - if (XLByteLT(t_thrd.xlog_cxt.receivedUpto, expectedRecPtr)) { + if (XLByteLE(t_thrd.xlog_cxt.receivedUpto, expectedRecPtr)) { t_thrd.xlog_cxt.failedSources = 0; /* @@ -19573,26 +19538,6 @@ static int SSDoradoReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePt SpinLockAcquire(&xlogctl->info_lck); xlogctl->IsRecoveryDone = true; SpinLockRelease(&xlogctl->info_lck); - static uint64 printFrequency = 0; - if (pg_atomic_read_u32(&t_thrd.walreceiverfuncs_cxt.WalRcv->rcvDoneFromShareStorage)) { - knl_g_set_redo_finish_status(REDO_FINISH_STATUS_LOCAL | REDO_FINISH_STATUS_CM); - if ((printFrequency & 0xFF) == 0) { - ereport(LOG, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), - errmsg("XLogPageRead set redo finish status," - "ReadRecPtr:%X/%X, EndRecPtr:%X/%X", - (uint32)(t_thrd.xlog_cxt.ReadRecPtr >> 32), - (uint32)(t_thrd.xlog_cxt.ReadRecPtr), - (uint32)(t_thrd.xlog_cxt.EndRecPtr >> 32), - (uint32)(t_thrd.xlog_cxt.EndRecPtr)))); - } - printFrequency++; - /* - * If it hasn't been long since last attempt, sleep 1s to - * avoid busy-waiting. - */ - pg_atomic_write_u32(&g_instance.comm_cxt.localinfo_cxt.need_disable_connection_node, false); - pg_usleep(2000000L); - } /* * If primary_conninfo is set, launch walreceiver to * try to stream the missing WAL, before retrying to @@ -19607,7 +19552,7 @@ static int SSDoradoReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePt */ load_server_mode(); - if (IS_SS_REPLICATION_MAIN_STANBY_NODE && CheckForFailoverTrigger()) { + if (SS_REPLICATION_MAIN_STANBY_NODE && CheckForFailoverTrigger()) { goto triggered; } ProcTxnWorkLoad(false); @@ -19615,7 +19560,7 @@ static int SSDoradoReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePt volatile WalRcvData *walrcv = t_thrd.walreceiverfuncs_cxt.WalRcv; rename_recovery_conf_for_roach(); - ereport(LOG, (errmsg("request xlog stream from shared storage at %X/%X.", + ereport(LOG, (errmsg("request xlog stream from dorado copy at %X/%X.", fetching_ckpt ? (uint32)(t_thrd.xlog_cxt.RedoStartLSN >> 32) : (uint32)(targetRecPtr >> 32), fetching_ckpt ? (uint32)t_thrd.xlog_cxt.RedoStartLSN @@ -19664,6 +19609,15 @@ static int SSDoradoReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePt */ RedoInterruptCallBack(); } + } else { + if (randAccess) { + t_thrd.xlog_cxt.curFileTLI = 0; + } + + sources = XLOG_FROM_PG_XLOG; + if (t_thrd.xlog_cxt.InArchiveRecovery) { + sources |= XLOG_FROM_ARCHIVE; + } } } @@ -19715,7 +19669,7 @@ int SSXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int re XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *readTLI, char* xlog_path) { int read_len; - if (SS_CLUSTER_DORADO_REPLICATION) { + if (SS_REPLICATION_DORADO_CLUSTER) { read_len = SSDoradoReadXLog(xlogreader, targetPagePtr, reqLen, targetRecPtr, readBuf, readTLI, xlog_path); } else { diff --git a/src/gausskernel/storage/file/fd.cpp b/src/gausskernel/storage/file/fd.cpp index 5ad4b4fc1..34db58e81 100755 --- a/src/gausskernel/storage/file/fd.cpp +++ b/src/gausskernel/storage/file/fd.cpp @@ -871,7 +871,7 @@ int BasicOpenFile(FileName fileName, int fileFlags, int fileMode) /* -* When SS_CLUSTER_DORADO_REPLICATION enabled, current xlog dictionary may be not the correct dictionary, +* When SS_REPLICATION_DORADO_CLUSTER enabled, current xlog dictionary may be not the correct dictionary, * because all xlog dictionaries are in the same LUN, we need loop over other dictionaries. */ int SSErgodicOpenXlogFile(XLogSegNo segno, int fileFlags, int fileMode) diff --git a/src/gausskernel/storage/ipc/procarray.cpp b/src/gausskernel/storage/ipc/procarray.cpp index defe71e22..a2bec4d0e 100755 --- a/src/gausskernel/storage/ipc/procarray.cpp +++ b/src/gausskernel/storage/ipc/procarray.cpp @@ -2364,7 +2364,7 @@ Snapshot GetSnapshotData(Snapshot snapshot, bool force_local_snapshot) } /* Check whether there's a standby requiring an older xmin when dms is enabled. */ - if (ENABLE_DMS && (SS_STANDBY_CLUSTER_NORMAL_MAIN_STANDBY || (SS_NORMAL_PRIMARY && IS_SS_REPLICATION_MAIN_STANBY_NODE))) { + if (SS_NORMAL_PRIMARY && SS_REPLICATION_MAIN_STANBY_NODE) { ss_xmin_info_t* xmin_info = &g_instance.dms_cxt.SSXminInfo; uint64 global_xmin = SSGetGlobalOldestXmin(u_sess->utils_cxt.RecentGlobalXmin); u_sess->utils_cxt.RecentGlobalXmin = global_xmin; diff --git a/src/gausskernel/storage/replication/basebackup.cpp b/src/gausskernel/storage/replication/basebackup.cpp index 6aebca5d5..9542a425e 100755 --- a/src/gausskernel/storage/replication/basebackup.cpp +++ b/src/gausskernel/storage/replication/basebackup.cpp @@ -30,6 +30,7 @@ #include "replication/dcf_data.h" #include "replication/walsender.h" #include "replication/walsender_private.h" +#include "replication/ss_cluster_replication.h" #include "replication/slot.h" #include "access/xlog.h" #include "storage/cfs/cfs_converter.h" @@ -2220,7 +2221,7 @@ static bool sendFile(char *readfilename, char *tarfilename, struct stat *statbuf ereport(ERROR, (errcode_for_file_access(), errmsg("could not read file \"%s\": %m", readfilename))); } } - if (g_instance.attr.attr_storage.enableIncrementalCheckpoint && isNeedCheck) { + if (g_instance.attr.attr_storage.enableIncrementalCheckpoint && isNeedCheck && !SS_REPLICATION_DORADO_CLUSTER) { uint32 segSize; GET_SEG_SIZE(undoFileType, segSize); /* len and cnt must be integer multiple of BLCKSZ. */ diff --git a/src/gausskernel/storage/replication/dataqueue.cpp b/src/gausskernel/storage/replication/dataqueue.cpp index 4424a1e2e..5957abdd8 100644 --- a/src/gausskernel/storage/replication/dataqueue.cpp +++ b/src/gausskernel/storage/replication/dataqueue.cpp @@ -301,7 +301,7 @@ DataQueuePtr PushToSenderQueue(const RelFileNode &rnode, BlockNumber blockNum, S LWLockRelease(DataSyncRepLock); if (g_instance.attr.attr_storage.max_wal_senders > 0) { - if (t_thrd.walsender_cxt.WalSndCtl->sync_master_standalone && !(IS_SHARED_STORAGE_MODE || SS_CLUSTER_DORADO_REPLICATION)) { + if (t_thrd.walsender_cxt.WalSndCtl->sync_master_standalone && !(IS_SHARED_STORAGE_MODE || SS_REPLICATION_DORADO_CLUSTER)) { ereport( LOG, (errmsg("failed to push rnode %u/%u/%u blockno %u into data-queue becuase sync_master_standalone " diff --git a/src/gausskernel/storage/replication/datasyncrep.cpp b/src/gausskernel/storage/replication/datasyncrep.cpp index 50aef20d9..c7acda72f 100644 --- a/src/gausskernel/storage/replication/datasyncrep.cpp +++ b/src/gausskernel/storage/replication/datasyncrep.cpp @@ -186,7 +186,7 @@ void WaitForDataSync(void) /* * if we modify the syncmode dynamically, we'll stop wait */ - if ((t_thrd.walsender_cxt.WalSndCtl->sync_master_standalone && !(IS_SHARED_STORAGE_MODE || SS_CLUSTER_DORADO_REPLICATION)) || + if ((t_thrd.walsender_cxt.WalSndCtl->sync_master_standalone && !(IS_SHARED_STORAGE_MODE || SS_REPLICATION_DORADO_CLUSTER)) || u_sess->attr.attr_storage.guc_synchronous_commit <= SYNCHRONOUS_COMMIT_LOCAL_FLUSH) { ereport(WARNING, (errmsg("canceling wait for synchronous replication due to syncmaster standalone."), diff --git a/src/gausskernel/storage/replication/libpqwalreceiver.cpp b/src/gausskernel/storage/replication/libpqwalreceiver.cpp index ebf0e1cde..736755914 100755 --- a/src/gausskernel/storage/replication/libpqwalreceiver.cpp +++ b/src/gausskernel/storage/replication/libpqwalreceiver.cpp @@ -239,7 +239,7 @@ static bool CheckRemoteServerSharedStorage(ServerMode remoteMode, PGresult* res) static bool CheckSSRemoteServerMode(ServerMode remoteMode, PGresult* res) { - if (IS_SS_REPLICATION_MAIN_STANBY_NODE) { + if (SS_REPLICATION_MAIN_STANBY_NODE) { if (remoteMode != PRIMARY_MODE) { PQclear(res); ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), @@ -493,7 +493,7 @@ ServerMode IdentifyRemoteMode() !t_thrd.walreceiver_cxt.AmWalReceiverForFailover && (!IS_PRIMARY_NORMAL(remoteMode)) && /* remoteMode of cascade standby is a standby */ - !t_thrd.xlog_cxt.is_cascade_standby && !(IS_SHARED_STORAGE_MODE || SS_CLUSTER_DORADO_REPLICATION)) { + !t_thrd.xlog_cxt.is_cascade_standby && !(IS_SHARED_STORAGE_MODE || SS_REPLICATION_DORADO_CLUSTER)) { PQclear(res); if (dummyStandbyMode) { @@ -506,7 +506,7 @@ ServerMode IdentifyRemoteMode() } if (t_thrd.postmaster_cxt.HaShmData->is_cascade_standby && remoteMode != STANDBY_MODE && - !(IS_SHARED_STORAGE_MODE || SS_CLUSTER_DORADO_REPLICATION)) { + !(IS_SHARED_STORAGE_MODE || SS_REPLICATION_DORADO_CLUSTER)) { PQclear(res); SpinLockAcquire(&walrcv->mutex); @@ -524,7 +524,7 @@ ServerMode IdentifyRemoteMode() } } - if (SS_CLUSTER_DORADO_REPLICATION && !CheckSSRemoteServerMode(remoteMode, res)) { + if (SS_REPLICATION_DORADO_CLUSTER && !CheckSSRemoteServerMode(remoteMode, res)) { return UNKNOWN_MODE; } @@ -577,7 +577,7 @@ static int32 IdentifyRemoteVersion() (errcode(ERRCODE_INVALID_STATUS), errmsg("could not get the local protocal version, make sure the PG_PROTOCOL_VERSION is defined"))); } - if (!IS_SHARED_STORAGE_STANDBY_CLUSTER_STANDBY_MODE && !IS_SS_REPLICATION_MAIN_STANBY_NODE) { + if (!IS_SHARED_STORAGE_STANDBY_CLUSTER_STANDBY_MODE && !SS_REPLICATION_MAIN_STANBY_NODE) { if (walrcv->conn_target != REPCONNTARGET_DUMMYSTANDBY && (localTerm == 0 || localTerm > remoteTerm) && !AM_HADR_WAL_RECEIVER) { PQclear(res); @@ -700,7 +700,7 @@ bool libpqrcv_connect(char *conninfo, XLogRecPtr *startpoint, char *slotname, in rc = memset_s(passwd, MAXPGPATH, 0, MAXPGPATH); securec_check(rc, "\0", "\0"); } else if (IS_SHARED_STORAGE_STANDBY_CLUSTER_STANDBY_MODE || - IS_SS_REPLICATION_MAIN_STANBY_NODE) { + SS_REPLICATION_MAIN_STANBY_NODE) { nRet = snprintf_s(conninfoRepl, sizeof(conninfoRepl), sizeof(conninfoRepl) - 1, "%s dbname=postgres replication=standby_cluster " "fallback_application_name=%s_hass " @@ -1417,7 +1417,7 @@ bool libpqrcv_receive(int timeout, unsigned char *type, char **buffer, int *len) } *type = *((unsigned char *)t_thrd.libwalreceiver_cxt.recvBuf); - if ((IS_SHARED_STORAGE_MODE || SS_CLUSTER_DORADO_REPLICATION) && !AM_HADR_WAL_RECEIVER && *type == 'w') { + if ((IS_SHARED_STORAGE_MODE || SS_REPLICATION_DORADO_CLUSTER) && !AM_HADR_WAL_RECEIVER && *type == 'w') { *len = 0; return false; } diff --git a/src/gausskernel/storage/replication/shared_storage_walreceiver.cpp b/src/gausskernel/storage/replication/shared_storage_walreceiver.cpp index a166f4caa..dbc14eb8a 100644 --- a/src/gausskernel/storage/replication/shared_storage_walreceiver.cpp +++ b/src/gausskernel/storage/replication/shared_storage_walreceiver.cpp @@ -369,7 +369,7 @@ bool shared_storage_connect(char *conninfo, XLogRecPtr *startpoint, char *slotna walrcv->peer_state = NORMAL_STATE; walrcv->isFirstTimeAccessStorage = true; - if (SS_CLUSTER_DORADO_REPLICATION) { + if (SS_REPLICATION_DORADO_CLUSTER) { libpgConnected = libpqrcv_connect(conninfo, startpoint, slotname, channel_identifier); return libpgConnected; } @@ -438,7 +438,7 @@ bool shared_storage_receive(int timeout, unsigned char *type, char **buffer, int * When ss cluster replication enabled, no xlog will receive, so return false directly. * Xlog will replicated by Dorado synchronous replication. */ - if (SS_CLUSTER_DORADO_REPLICATION) { + if (SS_REPLICATION_DORADO_CLUSTER) { return false; } @@ -447,7 +447,7 @@ bool shared_storage_receive(int timeout, unsigned char *type, char **buffer, int void shared_storage_send(const char *buffer, int nbytes) { - if (IS_SHARED_STORAGE_STANBY_MODE || SS_CLUSTER_DORADO_REPLICATION) { + if (IS_SHARED_STORAGE_STANBY_MODE || SS_REPLICATION_DORADO_CLUSTER) { if (t_thrd.libwalreceiver_cxt.streamConn) libpqrcv_send(buffer, nbytes); } diff --git a/src/gausskernel/storage/replication/ss_cluster_replication.cpp b/src/gausskernel/storage/replication/ss_cluster_replication.cpp index b747827c9..b3fb10268 100644 --- a/src/gausskernel/storage/replication/ss_cluster_replication.cpp +++ b/src/gausskernel/storage/replication/ss_cluster_replication.cpp @@ -162,7 +162,7 @@ void SSClusterDoradoStorageInit() void UpdateSSDoradoCtlInfoAndSync() { - if (!SS_CLUSTER_DORADO_REPLICATION) { + if (!SS_REPLICATION_DORADO_CLUSTER) { return; } @@ -174,7 +174,7 @@ void UpdateSSDoradoCtlInfoAndSync() bool CheckSSCtlInfoConsistency(XLogRecPtr localEnd) { - if (!SS_CLUSTER_DORADO_REPLICATION) { + if (!SS_REPLICATION_DORADO_CLUSTER) { return true; } @@ -208,7 +208,7 @@ void InitSSDoradoCtlInfo(ShareStorageXLogCtl *ctlInfo, uint64 sysidentifier) void CheckSSDoradoCtlInfo(XLogRecPtr localEnd) { - if (!SS_CLUSTER_DORADO_REPLICATION) { + if (!SS_REPLICATION_DORADO_CLUSTER) { return; } diff --git a/src/gausskernel/storage/replication/syncrep.cpp b/src/gausskernel/storage/replication/syncrep.cpp index 356fe9ea8..eab0c0693 100755 --- a/src/gausskernel/storage/replication/syncrep.cpp +++ b/src/gausskernel/storage/replication/syncrep.cpp @@ -56,6 +56,7 @@ #include "replication/walsender.h" #include "replication/walsender_private.h" #include "replication/shared_storage_walreceiver.h" +#include "replication/ss_cluster_replication.h" #include "storage/pmsignal.h" #include "storage/proc.h" #include "tcop/tcopprot.h" @@ -247,8 +248,8 @@ SyncWaitRet SyncRepWaitForLSN(XLogRecPtr XactCommitLSN, bool enableHandleCancel) RESUME_INTERRUPTS(); return REPSYNCED; } - if (t_thrd.walsender_cxt.WalSndCtl->sync_master_standalone && !IS_SHARED_STORAGE_MODE && - !DelayIntoMostAvaSync(false)) { + if (t_thrd.walsender_cxt.WalSndCtl->sync_master_standalone && !DelayIntoMostAvaSync(false) && + !IS_SHARED_STORAGE_MODE && !SS_REPLICATION_DORADO_CLUSTER) { LWLockRelease(SyncRepLock); RESUME_INTERRUPTS(); return STAND_ALONE; diff --git a/src/gausskernel/storage/replication/walreceiver.cpp b/src/gausskernel/storage/replication/walreceiver.cpp index 470373c8d..9a2102421 100755 --- a/src/gausskernel/storage/replication/walreceiver.cpp +++ b/src/gausskernel/storage/replication/walreceiver.cpp @@ -388,7 +388,7 @@ void WalRcvrProcessData(TimestampTz *last_recv_timestamp, bool *ping_sent) } /* walrec in dorado replication mode not need walrecwrite */ - if (!IS_SS_REPLICATION_MAIN_STANBY_NODE && !WalRcvWriterInProgress()) + if (!SS_REPLICATION_MAIN_STANBY_NODE && !WalRcvWriterInProgress()) ereport(FATAL, (errmsg("terminating walreceiver process due to the death of walrcvwriter"))); #ifndef ENABLE_MULTIPLE_NODES /* For Paxos, receive wal should be done by send log callback function */ @@ -470,9 +470,7 @@ void WalReceiverMain(void) int nRet = 0; errno_t rc = 0; - if ((ENABLE_DSS && t_thrd.postmaster_cxt.HaShmData->current_mode == STANDBY_MODE && - g_instance.attr.attr_common.cluster_run_mode == RUN_MODE_STANDBY && - g_instance.attr.attr_storage.xlog_file_path != 0) || IS_SS_REPLICATION_MAIN_STANBY_NODE) { + if (SS_REPLICATION_MAIN_STANBY_NODE) { ereport(LOG, (errmsg("walreceiver thread started for main standby"))); } else { Assert(ENABLE_DSS == false); @@ -579,7 +577,7 @@ void WalReceiverMain(void) int walreplindex = hashmdata->current_repl; SpinLockRelease(&hashmdata->mutex); - if (!IS_SHARED_STORAGE_STANDBY_CLUSTER_STANDBY_MODE && !IS_SS_REPLICATION_MAIN_STANBY_NODE) { + if (!IS_SHARED_STORAGE_STANDBY_CLUSTER_STANDBY_MODE && !SS_REPLICATION_MAIN_STANBY_NODE) { replConnInfo = t_thrd.postmaster_cxt.ReplConnArray[walreplindex]; } else if (walreplindex >= MAX_REPLNODE_NUM) { replConnInfo = t_thrd.postmaster_cxt.CrossClusterReplConnArray[walreplindex - MAX_REPLNODE_NUM]; @@ -849,7 +847,7 @@ bool HasBuildReason() static void rcvAllXlog() { - if (SS_CLUSTER_DORADO_REPLICATION) { + if (SS_REPLICATION_DORADO_CLUSTER) { return; } @@ -890,7 +888,7 @@ static void WalRcvDie(int code, Datum arg) { /* use volatile pointer to prevent code rearrangement */ volatile WalRcvData *walrcv = t_thrd.walreceiverfuncs_cxt.WalRcv; - if ((IS_SHARED_STORAGE_MODE || SS_CLUSTER_DORADO_REPLICATION) && !t_thrd.walreceiver_cxt.termChanged) { + if ((IS_SHARED_STORAGE_MODE || SS_REPLICATION_DORADO_CLUSTER) && !t_thrd.walreceiver_cxt.termChanged) { SpinLockAcquire(&walrcv->mutex); walrcv->walRcvState = WALRCV_STOPPING; SpinLockRelease(&walrcv->mutex); @@ -1134,10 +1132,10 @@ static void XLogWalRcvProcessMsg(unsigned char type, char *buf, Size len) { #ifndef ENABLE_MULTIPLE_NODES if (IS_SHARED_STORAGE_STANDBY_CLUSTER_STANDBY_MODE || AM_HADR_WAL_RECEIVER - || SS_CLUSTER_DORADO_REPLICATION) { + || SS_REPLICATION_DORADO_CLUSTER) { #else if (IS_SHARED_STORAGE_STANDBY_CLUSTER_STANDBY_MODE || AM_HADR_WAL_RECEIVER - || SS_CLUSTER_DORADO_REPLICATION || AM_HADR_CN_WAL_RECEIVER) { + || SS_REPLICATION_DORADO_CLUSTER || AM_HADR_CN_WAL_RECEIVER) { #endif break; } @@ -1566,7 +1564,7 @@ void XLogWalRcvSendReply(bool force, bool requestReply) ReadShareStorageCtlInfo(ctlInfo); receivePtr = ctlInfo->insertHead; AlignFreeShareStorageCtl(ctlInfo); - } else if (SS_CLUSTER_DORADO_REPLICATION) { + } else if (SS_REPLICATION_DORADO_CLUSTER) { ReadSSDoradoCtlInfoFile(); receivePtr = g_instance.xlog_cxt.ssReplicationXLogCtl->insertHead; writePtr = g_instance.xlog_cxt.ssReplicationXLogCtl->insertHead; @@ -1616,7 +1614,7 @@ void XLogWalRcvSendReply(bool force, bool requestReply) t_thrd.walreceiver_cxt.reply_message->replyRequested = requestReply; SpinLockAcquire(&hashmdata->mutex); - if (!IS_SHARED_STORAGE_STANDBY_CLUSTER_STANDBY_MODE && !IS_SS_REPLICATION_MAIN_STANBY_NODE) + if (!IS_SHARED_STORAGE_STANDBY_CLUSTER_STANDBY_MODE && !SS_REPLICATION_MAIN_STANBY_NODE) t_thrd.walreceiver_cxt.reply_message->peer_role = hashmdata->current_mode; else t_thrd.walreceiver_cxt.reply_message->peer_role = STANDBY_CLUSTER_MODE; @@ -2297,7 +2295,7 @@ Datum pg_stat_get_wal_receiver(PG_FUNCTION_ARGS) sndReplay = sendLocFix; } - if (SS_CLUSTER_DORADO_REPLICATION) { + if (SS_REPLICATION_DORADO_CLUSTER) { ReadSSDoradoCtlInfoFile(); XLogRecPtr sendLocFix = rcvReceived; ShareStorageXLogCtl *ctlInfo = g_instance.xlog_cxt.ssReplicationXLogCtl; diff --git a/src/gausskernel/storage/replication/walreceiverfuncs.cpp b/src/gausskernel/storage/replication/walreceiverfuncs.cpp index b56cc0c0a..82d7a12df 100755 --- a/src/gausskernel/storage/replication/walreceiverfuncs.cpp +++ b/src/gausskernel/storage/replication/walreceiverfuncs.cpp @@ -260,7 +260,7 @@ static void SetWalRcvConninfo(ReplConnTarget conn_target) SpinLockRelease(&walrcv->mutex); ereport(LOG, (errmsg("wal receiver try to connect to %s index %d .", walrcv->conninfo, useIndex))); SpinLockAcquire(&hashmdata->mutex); - if (!IS_SHARED_STORAGE_STANDBY_CLUSTER_STANDBY_MODE && !IS_SS_REPLICATION_MAIN_STANBY_NODE) + if (!IS_SHARED_STORAGE_STANDBY_CLUSTER_STANDBY_MODE && !SS_REPLICATION_MAIN_STANBY_NODE) hashmdata->current_repl = useIndex; else hashmdata->current_repl = MAX_REPLNODE_NUM + useIndex; @@ -417,7 +417,7 @@ static void set_rcv_slot_name(const char *slotname) SpinLockAcquire(&hashmdata->mutex); replIdx = hashmdata->current_repl; SpinLockRelease(&hashmdata->mutex); - if ((IS_SHARED_STORAGE_STANDBY_CLUSTER_STANDBY_MODE || IS_SS_REPLICATION_MAIN_STANBY_NODE) + if ((IS_SHARED_STORAGE_STANDBY_CLUSTER_STANDBY_MODE || SS_REPLICATION_MAIN_STANBY_NODE) && replIdx >= MAX_REPLNODE_NUM) { replIdx = replIdx - MAX_REPLNODE_NUM; } @@ -893,7 +893,7 @@ ReplConnInfo *GetRepConnArray(int *cur_idx) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid replication node index:%d", *cur_idx))); } - if (!IS_SHARED_STORAGE_STANDBY_CLUSTER_STANDBY_MODE && !IS_SS_REPLICATION_MAIN_STANBY_NODE) + if (!IS_SHARED_STORAGE_STANDBY_CLUSTER_STANDBY_MODE && !SS_REPLICATION_MAIN_STANBY_NODE) replConnInfoArray = &t_thrd.postmaster_cxt.ReplConnArray[0]; else replConnInfoArray = &t_thrd.postmaster_cxt.CrossClusterReplConnArray[0]; diff --git a/src/gausskernel/storage/replication/walsender.cpp b/src/gausskernel/storage/replication/walsender.cpp index 178e8b97c..6d4a2ed3e 100755 --- a/src/gausskernel/storage/replication/walsender.cpp +++ b/src/gausskernel/storage/replication/walsender.cpp @@ -136,8 +136,8 @@ static int g_appname_extra_len = 3; /* [+]+\0 */ #define AmWalSenderToStandby() (t_thrd.walsender_cxt.MyWalSnd->sendRole == SNDROLE_PRIMARY_STANDBY) #define USE_PHYSICAL_XLOG_SEND \ - (AM_WAL_HADR_SENDER || !SS_CLUSTER_DORADO_REPLICATION || !IS_SHARED_STORAGE_MODE || (walsnd->sendRole == SNDROLE_PRIMARY_BUILDSTANDBY)) -#define USE_SYNC_REP_FLUSH_PTR (AM_WAL_HADR_SENDER && (!IS_SHARED_STORAGE_MODE && !SS_CLUSTER_DORADO_REPLICATION)) + (AM_WAL_HADR_SENDER || !SS_REPLICATION_DORADO_CLUSTER || !IS_SHARED_STORAGE_MODE || (walsnd->sendRole == SNDROLE_PRIMARY_BUILDSTANDBY)) +#define USE_SYNC_REP_FLUSH_PTR (AM_WAL_HADR_SENDER && (!IS_SHARED_STORAGE_MODE && !SS_REPLICATION_DORADO_CLUSTER)) /* Statistics for log control */ static const int MICROSECONDS_PER_SECONDS = 1000000; @@ -2645,7 +2645,7 @@ static void LogCtrlDoActualSleep(volatile WalSnd *walsnd, bool forceUpdate) u_sess->attr.attr_storage.hadr_recovery_point_target > 0) { LogCtrlExecuteSleeping(walsnd, forceUpdate, logical_slot_sleep_flag); } else { - if (logical_slot_sleep_flag && !IS_SHARED_STORAGE_MODE) { + if (logical_slot_sleep_flag && !IS_SHARED_STORAGE_MODE && !SS_REPLICATION_DORADO_CLUSTER) { pg_usleep(g_logical_slot_sleep_time); } } @@ -2653,7 +2653,7 @@ static void LogCtrlDoActualSleep(volatile WalSnd *walsnd, bool forceUpdate) if (u_sess->attr.attr_storage.target_rto > 0) { LogCtrlExecuteSleeping(walsnd, forceUpdate, logical_slot_sleep_flag); } else { - if (logical_slot_sleep_flag && !IS_SHARED_STORAGE_MODE) { + if (logical_slot_sleep_flag && !IS_SHARED_STORAGE_MODE && !SS_REPLICATION_DORADO_CLUSTER) { pg_usleep(g_logical_slot_sleep_time); } } @@ -2684,7 +2684,7 @@ static void LogCtrlExecuteSleeping(volatile WalSnd *walsnd, bool forceUpdate, bo } LogCtrlSleep(); if (logicalSlotSleepFlag && g_logical_slot_sleep_time > t_thrd.walsender_cxt.MyWalSnd->log_ctrl.sleep_time && - !IS_SHARED_STORAGE_MODE) { + !IS_SHARED_STORAGE_MODE && !SS_REPLICATION_DORADO_CLUSTER) { pg_usleep(g_logical_slot_sleep_time - t_thrd.walsender_cxt.MyWalSnd->log_ctrl.sleep_time); } } @@ -2925,7 +2925,7 @@ static void ProcessStandbyReplyMessage(void) * because primary xlog will cover standby xlog by Dorado synchronous replication. * 2. Otherwise, we only need to confirm that standby xlog has been flushed successfully. */ - if (SS_CLUSTER_DORADO_REPLICATION) { + if (SS_REPLICATION_DORADO_CLUSTER) { AdvanceReplicationSlot(reply.apply); } else { AdvanceReplicationSlot(reply.flush); @@ -3167,7 +3167,7 @@ static void LogCtrlCountSleepLimit(void) static void LogCtrlSleep(void) { volatile WalSnd *walsnd = t_thrd.walsender_cxt.MyWalSnd; - if (IS_SHARED_STORAGE_MODE) { + if (IS_SHARED_STORAGE_MODE || SS_REPLICATION_DORADO_CLUSTER) { if (walsnd->log_ctrl.sleep_time > MICROSECONDS_PER_SECONDS) { walsnd->log_ctrl.sleep_time = MICROSECONDS_PER_SECONDS; } @@ -3558,7 +3558,7 @@ static void LogCtrlCalculateCurrentRPO(StandbyReplyMessage *reply) if (AM_WAL_HADR_CN_SENDER) { flushPtr = GetFlushRecPtr(); } else if (AM_WAL_SHARE_STORE_SENDER) { - if (SS_CLUSTER_DORADO_REPLICATION) { + if (SS_REPLICATION_DORADO_CLUSTER) { flushPtr = g_instance.xlog_cxt.ssReplicationXLogCtl->insertHead; } else { flushPtr = g_instance.xlog_cxt.shareStorageXLogCtl->insertHead; @@ -4200,7 +4200,7 @@ static int WalSndLoop(WalSndSendDataCallback send_data) XLByteEQ(t_thrd.walsender_cxt.sentPtr, t_thrd.walsender_cxt.MyWalSnd->flush)) t_thrd.walsender_cxt.walsender_shutdown_requested = true; } - if (IS_SHARED_STORAGE_MODE) { + if (IS_SHARED_STORAGE_MODE || SS_REPLICATION_DORADO_CLUSTER) { t_thrd.walsender_cxt.walsender_shutdown_requested = true; } } @@ -6278,7 +6278,7 @@ Datum pg_stat_get_wal_senders(PG_FUNCTION_ARGS) AlignFreeShareStorageCtl(ctlInfo); } - if (SS_CLUSTER_DORADO_REPLICATION && !AM_WAL_HADR_SENDER) { + if (SS_REPLICATION_DORADO_CLUSTER && !AM_WAL_HADR_SENDER) { ReadSSDoradoCtlInfoFile(); ShareStorageXLogCtl *ctlInfo = g_instance.xlog_cxt.ssReplicationXLogCtl; sentRecPtr = ctlInfo->insertHead; diff --git a/src/gausskernel/storage/smgr/segstore.cpp b/src/gausskernel/storage/smgr/segstore.cpp index 7d5aae9e0..0c8bef948 100755 --- a/src/gausskernel/storage/smgr/segstore.cpp +++ b/src/gausskernel/storage/smgr/segstore.cpp @@ -370,8 +370,8 @@ SegPageLocation seg_logic_to_physic_mapping(SMgrRelation reln, SegmentHead *seg_ BlockNumber blocknum; /* Recovery thread should use physical location to read data directly. */ - if (SS_STANDBY_CLUSTER_MAIN_STANDBY || IS_SS_REPLICATION_MAIN_STANBY_NODE) { - ereport(DEBUG1, (errmsg("can segment address translation when role is IS_SS_REPLICATION_MAIN_STANBY_NODE"))); + if (SS_REPLICATION_MAIN_STANBY_NODE) { + ereport(DEBUG1, (errmsg("can segment address translation when role is SS_REPLICATION_MAIN_STANBY_NODE"))); } else { if (RecoveryInProgress() && !CurrentThreadIsWorker() && !SS_IN_FLUSHCOPY) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("recovery is in progress"), diff --git a/src/gausskernel/storage/xlog_share_storage/xlog_share_storage.cpp b/src/gausskernel/storage/xlog_share_storage/xlog_share_storage.cpp index 99fc88d4a..7517256d7 100644 --- a/src/gausskernel/storage/xlog_share_storage/xlog_share_storage.cpp +++ b/src/gausskernel/storage/xlog_share_storage/xlog_share_storage.cpp @@ -139,16 +139,7 @@ void LocalXLogRead(char *buf, XLogRecPtr startptr, Size count) } XLByteToSeg(recptr, t_thrd.sharestoragexlogcopyer_cxt.readSegNo); - if (SS_STANDBY_FAILOVER && SS_PRIMARY_CLUSTER_STANDBY) { - int nRet; - nRet = snprintf_s(path, MAXPGPATH, MAXPGPATH - 1, "%s/%08X%08X%08X", - g_instance.dms_cxt.SSRecoveryInfo.recovery_xlog_dir, t_thrd.xlog_cxt.ThisTimeLineID, - (uint32)((t_thrd.sharestoragexlogcopyer_cxt.readSegNo) / XLogSegmentsPerXLogId), - (uint32)((t_thrd.sharestoragexlogcopyer_cxt.readSegNo) % XLogSegmentsPerXLogId)); - securec_check_ss(nRet, "\0", "\0"); - } else { - XLogFilePath(path, MAXPGPATH, t_thrd.xlog_cxt.ThisTimeLineID, t_thrd.sharestoragexlogcopyer_cxt.readSegNo); - } + XLogFilePath(path, MAXPGPATH, t_thrd.xlog_cxt.ThisTimeLineID, t_thrd.sharestoragexlogcopyer_cxt.readSegNo); t_thrd.sharestoragexlogcopyer_cxt.readFile = BasicOpenFile(path, O_RDONLY | PG_BINARY, 0); if (t_thrd.sharestoragexlogcopyer_cxt.readFile < 0) { diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h index f9eef7e32..be17c9ed6 100755 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@ -93,14 +93,6 @@ extern volatile uint64 sync_system_identifier; #define XLOG_FROM_PG_XLOG (1 << 1) /* Existing file in pg_xlog */ #define XLOG_FROM_STREAM (1 << 2) /* Streamed from master */ -#define DORADO_STANDBY_CLUSTER (g_instance.attr.attr_common.cluster_run_mode == RUN_MODE_STANDBY && \ - g_instance.attr.attr_storage.xlog_file_path != NULL) -#define DORADO_PRIMARY_CLUSTER (g_instance.attr.attr_common.cluster_run_mode == RUN_MODE_PRIMARY && \ - g_instance.attr.attr_storage.xlog_file_path != NULL) -#define DORADO_STANDBY_CLUSTER_MAINSTANDBY_NODE ((t_thrd.postmaster_cxt.HaShmData->current_mode == STANDBY_MODE) && \ - (g_instance.attr.attr_common.cluster_run_mode == RUN_MODE_STANDBY) && \ - (g_instance.attr.attr_storage.xlog_file_path != NULL)) - /* * Recovery target type. * Only set during a Point in Time recovery, not when standby_mode = on diff --git a/src/include/ddes/dms/ss_common_attr.h b/src/include/ddes/dms/ss_common_attr.h index 59b933d2f..f6def8807 100644 --- a/src/include/ddes/dms/ss_common_attr.h +++ b/src/include/ddes/dms/ss_common_attr.h @@ -98,57 +98,6 @@ /* Mode in dorado hyperreplication and dms enabled as follow */ -/* main standby in standby cluster */ -#define SS_STANDBY_CLUSTER_MAIN_STANDBY \ - (ENABLE_DMS && (t_thrd.xlog_cxt.server_mode == STANDBY_MODE || \ - t_thrd.postmaster_cxt.HaShmData->current_mode == STANDBY_MODE) && \ - (g_instance.attr.attr_common.cluster_run_mode == RUN_MODE_STANDBY) && \ - (g_instance.attr.attr_storage.xlog_file_path != NULL)) - -/* standby mode in primary or standby cluster */ -#define SS_PRIMARY_STANDBY_CLUSTER_STANDBY \ - (ENABLE_DMS && (t_thrd.xlog_cxt.server_mode == NORMAL_MODE || \ - t_thrd.postmaster_cxt.HaShmData->current_mode == NORMAL_MODE) && \ - (g_instance.attr.attr_storage.xlog_file_path != NULL)) - -/* standby mode in primary cluster */ -#define SS_PRIMARY_CLUSTER_STANDBY \ - (ENABLE_DMS && (t_thrd.xlog_cxt.server_mode == NORMAL_MODE || \ - t_thrd.postmaster_cxt.HaShmData->current_mode == NORMAL_MODE) && \ - (g_instance.attr.attr_common.cluster_run_mode == RUN_MODE_PRIMARY) && \ - (g_instance.attr.attr_storage.xlog_file_path != NULL)) - -/* arbitrary mode when dorado hyperreplication and dms enabled */ -#define SS_PRIMARY_STANDBY_CLUSTER_NORMAL \ - (ENABLE_DMS && ((g_instance.attr.attr_common.cluster_run_mode == RUN_MODE_PRIMARY) || \ - (g_instance.attr.attr_common.cluster_run_mode == RUN_MODE_STANDBY)) && \ - (g_instance.attr.attr_storage.xlog_file_path != NULL)) - -/* primary mode in primary cluster, after reform done and primary id has been determined */ -#define SS_PRIMARY_CLUSTER_NORMAL_PRIMARY \ - (SS_NORMAL_PRIMARY && (t_thrd.xlog_cxt.server_mode == PRIMARY_MODE || \ - t_thrd.postmaster_cxt.HaShmData->current_mode == PRIMARY_MODE) && \ - (g_instance.attr.attr_common.cluster_run_mode == RUN_MODE_PRIMARY) && \ - (g_instance.attr.attr_storage.xlog_file_path != NULL)) - -/* main standby in standby cluster, after reform done and primary id has been determined */ -#define SS_STANDBY_CLUSTER_NORMAL_MAIN_STANDBY \ - (SS_NORMAL_PRIMARY && (t_thrd.xlog_cxt.server_mode == STANDBY_MODE || \ - t_thrd.postmaster_cxt.HaShmData->current_mode == STANDBY_MODE) && \ - (g_instance.attr.attr_common.cluster_run_mode == RUN_MODE_STANDBY) && \ - (g_instance.attr.attr_storage.xlog_file_path != NULL)) - -/* standby mode in standby cluster, after reform done and primary id has been determined */ -#define SS_STANDBY_CLUSTER_NORMAL_STANDBY \ - (SS_NORMAL_STANDBY && (t_thrd.xlog_cxt.server_mode == STANDBY_MODE || \ - t_thrd.postmaster_cxt.HaShmData->current_mode == STANDBY_MODE) && \ - (g_instance.attr.attr_common.cluster_run_mode == RUN_MODE_STANDBY) && \ - (g_instance.attr.attr_storage.xlog_file_path != NULL)) - -/* standby mode in primary or standby, after reform done and primary id has been determined */ -#define SS_PRIMARY_STANDBY_CLUSTER_NORMAL_STANDBY \ - (SS_NORMAL_STANDBY && (g_instance.attr.attr_storage.xlog_file_path != NULL)) - #define SS_CLUSTER_ONDEMAND_NOT_NORAML \ (ENABLE_DMS && (g_instance.dms_cxt.SSReformerControl.clusterStatus != CLUSTER_NORMAL)) #define SS_CLUSTER_ONDEMAND_BUILD \ diff --git a/src/include/ddes/dms/ss_dms_recovery.h b/src/include/ddes/dms/ss_dms_recovery.h index 308ac92d8..a763ac4f3 100644 --- a/src/include/ddes/dms/ss_dms_recovery.h +++ b/src/include/ddes/dms/ss_dms_recovery.h @@ -77,6 +77,7 @@ typedef struct ss_recovery_info { volatile failover_ckpt_status_t failover_ckpt_status; char recovery_xlog_dir[MAXPGPATH]; int recovery_inst_id; + int instid_list[DMS_MAX_INSTANCE]; LWLock* update_seg_lock; bool new_primary_reset_walbuf_flag; bool ready_to_startup; // when DB start (except failover), the flag will set true diff --git a/src/include/ddes/dms/ss_reform_common.h b/src/include/ddes/dms/ss_reform_common.h index 3d7118340..227d762e8 100644 --- a/src/include/ddes/dms/ss_reform_common.h +++ b/src/include/ddes/dms/ss_reform_common.h @@ -41,6 +41,7 @@ int SSReadXlogInternal(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, XL int readLen); XLogReaderState *SSXLogReaderAllocate(XLogPageReadCB pagereadfunc, void *private_data, Size alignedSize); void SSGetRecoveryXlogPath(); +void SSDoradoGetInstidList(); void SSSaveReformerCtrl(bool force = false); void SSReadControlFile(int id, bool updateDmsCtx = false); void SSClearSegCache(); diff --git a/src/include/replication/ss_cluster_replication.h b/src/include/replication/ss_cluster_replication.h index 70a0b4c21..17b81ca54 100644 --- a/src/include/replication/ss_cluster_replication.h +++ b/src/include/replication/ss_cluster_replication.h @@ -32,36 +32,36 @@ const uint32 SS_DORADO_CTL_INFO_SIZE = 512; -#define SS_CLUSTER_DORADO_REPLICATION \ +#define SS_REPLICATION_DORADO_CLUSTER \ (ENABLE_DSS && g_instance.attr.attr_storage.enable_ss_dorado) /* Primary Cluster in SS replication */ -#define SS_REPLICATION_MAIN_CLUSTER \ - (SS_CLUSTER_DORADO_REPLICATION && (g_instance.attr.attr_common.cluster_run_mode == RUN_MODE_PRIMARY)) +#define SS_REPLICATION_PRIMARY_CLUSTER \ + (SS_REPLICATION_DORADO_CLUSTER && (g_instance.attr.attr_common.cluster_run_mode == RUN_MODE_PRIMARY)) /* Standby Cluster in SS replication */ #define SS_REPLICATION_STANDBY_CLUSTER \ - (SS_CLUSTER_DORADO_REPLICATION && (g_instance.attr.attr_common.cluster_run_mode == RUN_MODE_STANDBY)) + (SS_REPLICATION_DORADO_CLUSTER && (g_instance.attr.attr_common.cluster_run_mode == RUN_MODE_STANDBY)) /* Primary node in SS replication, means primary node in main cluster. */ -#define IS_SS_REPLICATION_PRIMARY_NODE \ - (SS_REPLICATION_MAIN_CLUSTER && (t_thrd.postmaster_cxt.HaShmData->current_mode == PRIMARY_MODE)) +#define SS_REPLICATION_PRIMARY_NODE \ + (SS_REPLICATION_PRIMARY_CLUSTER && (t_thrd.postmaster_cxt.HaShmData->current_mode == PRIMARY_MODE)) /* Standby node in SS replication, means standby node in main cluster. */ -#define IS_SS_REPLICATION_PRIMARY_CLUSTER_STANDBY_NODE \ - (SS_REPLICATION_MAIN_CLUSTER && (t_thrd.postmaster_cxt.HaShmData->current_mode == NORMAL_MODE)) +#define SS_REPLICATION_PRIMARY_CLUSTER_STANDBY_NODE \ + (SS_REPLICATION_PRIMARY_CLUSTER && (t_thrd.postmaster_cxt.HaShmData->current_mode == NORMAL_MODE)) /* Main standby node in SS replication, means primary node in standby cluster. */ -#define IS_SS_REPLICATION_MAIN_STANBY_NODE \ +#define SS_REPLICATION_MAIN_STANBY_NODE \ (SS_REPLICATION_STANDBY_CLUSTER && (t_thrd.postmaster_cxt.HaShmData->current_mode == STANDBY_MODE)) /* Standby node in SS replication, means standby node in standby cluster. */ -#define IS_SS_REPLICATION_STANDBY_CLUSTER_STANDBY_NODE \ +#define SS_REPLICATION_STANDBY_CLUSTER_STANDBY_NODE \ (SS_REPLICATION_STANDBY_CLUSTER && (t_thrd.postmaster_cxt.HaShmData->current_mode == NORMAL_MODE)) /* All Standby in SS replication, means nodes other than primary node in primary cluster and standby cluster */ -#define IS_SS_REPLICATION_STANBY_NODE \ - (SS_CLUSTER_DORADO_REPLICATION && (t_thrd.postmaster_cxt.HaShmData->current_mode == NORMAL_MODE)) +#define SS_REPLICATION_STANBY_NODE \ + (SS_REPLICATION_DORADO_CLUSTER && (t_thrd.postmaster_cxt.HaShmData->current_mode == NORMAL_MODE)) void SSClusterDoradoStorageInit(); diff --git a/src/include/tool_common.h b/src/include/tool_common.h index 4f0b57eb5..3f84091ce 100644 --- a/src/include/tool_common.h +++ b/src/include/tool_common.h @@ -111,11 +111,11 @@ typedef struct DssOptions { bool enable_dss; int instance_id; int primaryInstId; + int interNodeNum; char *vgname; char *vglog; char *vgdata; char *socketpath; - int interNodeNum; } DssOptions; typedef struct SSInstanceConfig { From c63c8980b80d10045faf2a25b7837d7d2bfb2ffe Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Tue, 22 Aug 2023 20:40:29 +0800 Subject: [PATCH 153/304] =?UTF-8?q?dss=E6=8E=A8=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/ddes/ddes_commit_id | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gausskernel/ddes/ddes_commit_id b/src/gausskernel/ddes/ddes_commit_id index 7c95953f9..ffb7f2e20 100644 --- a/src/gausskernel/ddes/ddes_commit_id +++ b/src/gausskernel/ddes/ddes_commit_id @@ -1,3 +1,3 @@ dms_commit_id=b12ceed7bb0d3de049d8b4858180c77110ff6444 -dss_commit_id=0b9c9bc1a2c3cac463ea70fd6abc1790e9f125aa +dss_commit_id=45ae7916cbdda2b2e64c02811db30df565cf34fa cbb_commit_id=d45b3bdee26012a8d14adf1f536a2d8a24a177fe From 5a8c2f593f1664b681f30868963e621aab3b8efe Mon Sep 17 00:00:00 2001 From: dongning12 Date: Thu, 17 Aug 2023 22:23:32 +0800 Subject: [PATCH 154/304] =?UTF-8?q?=E3=80=90=E8=B5=84=E6=BA=90=E6=B1=A0?= =?UTF-8?q?=E5=8C=96=E3=80=91=E9=80=82=E9=85=8DDMS=E6=8A=A5=E6=96=87?= =?UTF-8?q?=E5=85=BC=E5=AE=B9=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ddes/adapter/ss_dms_callback.cpp | 11 +++- src/gausskernel/ddes/ddes_commit_id | 4 +- src/include/ddes/dms/dms_api.h | 57 ++++++++++++++++--- 3 files changed, 59 insertions(+), 13 deletions(-) diff --git a/src/gausskernel/ddes/adapter/ss_dms_callback.cpp b/src/gausskernel/ddes/adapter/ss_dms_callback.cpp index 05f160755..5b51d719c 100644 --- a/src/gausskernel/ddes/adapter/ss_dms_callback.cpp +++ b/src/gausskernel/ddes/adapter/ss_dms_callback.cpp @@ -1017,9 +1017,12 @@ static int32 CBProcessReleaseAllLock(uint32 len) return res; } -static int32 CBProcessBroadcast(void *db_handle, char *data, unsigned int len, char *output_msg, - uint32 *output_msg_len) +static int32 CBProcessBroadcast(void *db_handle, dms_broadcast_context_t *broad_ctx) { + char *data = broad_ctx->data; + unsigned int len = broad_ctx->len; + char *output_msg = broad_ctx->output_msg; + unsigned int *output_msg_len = broad_ctx->output_msg_len; int32 ret = DMS_SUCCESS; SSBroadcastOp bcast_op = *(SSBroadcastOp *)data; @@ -1080,8 +1083,10 @@ static int32 CBProcessBroadcast(void *db_handle, char *data, unsigned int len, c return ret; } -static int32 CBProcessBroadcastAck(void *db_handle, char *data, unsigned int len) +static int32 CBProcessBroadcastAck(void *db_handle, dms_broadcast_context_t *broad_ctx) { + char *data = broad_ctx->data; + unsigned int len = broad_ctx->len; int32 ret = DMS_SUCCESS; SSBroadcastOpAck bcast_op = *(SSBroadcastOpAck *)data; diff --git a/src/gausskernel/ddes/ddes_commit_id b/src/gausskernel/ddes/ddes_commit_id index ffb7f2e20..514b56258 100644 --- a/src/gausskernel/ddes/ddes_commit_id +++ b/src/gausskernel/ddes/ddes_commit_id @@ -1,3 +1,3 @@ -dms_commit_id=b12ceed7bb0d3de049d8b4858180c77110ff6444 +dms_commit_id=19301cb7b3dd7f959cf6269c6d2aa8c8f23804fa dss_commit_id=45ae7916cbdda2b2e64c02811db30df565cf34fa -cbb_commit_id=d45b3bdee26012a8d14adf1f536a2d8a24a177fe +cbb_commit_id=41cadda4643656bcd856bd37d2b6908e44a5d46c \ No newline at end of file diff --git a/src/include/ddes/dms/dms_api.h b/src/include/ddes/dms/dms_api.h index 513e58f7b..94524a99a 100644 --- a/src/include/ddes/dms/dms_api.h +++ b/src/include/ddes/dms/dms_api.h @@ -200,12 +200,24 @@ typedef struct st_dms_xmap_ctx { unsigned int dest_id; } dms_xmap_ctx_t; +typedef struct st_dms_process_context { + void *db_handle; + unsigned int sess_id; // current session id + unsigned int rmid; // current rm id + unsigned int inst_id; // current instance id +} dms_process_context_t; + typedef struct st_dms_context { - unsigned int inst_id; // current instance id - unsigned int sess_id; // current session id - unsigned int rmid; // current rm id + union { + struct { + void *db_handle; + unsigned int sess_id; // current session id + unsigned int rmid; // current rm id + unsigned int inst_id; // current instance id + }; + dms_process_context_t proc_ctx; + }; dms_session_e sess_type; // request page: recovery session flag - void *db_handle; unsigned char is_try; unsigned char type; unsigned short len; @@ -293,6 +305,14 @@ typedef struct st_dms_edp_info { }; } dms_edp_info_t; +typedef struct st_dms_broadcast_context { + char *data; + unsigned int len; + char *output_msg; + unsigned int *output_msg_len; + unsigned int msg_version; +} dms_broadcast_context_t; + typedef struct st_dms_buf_ctrl { volatile unsigned char is_remote_dirty; volatile unsigned char lock_mode; // used only in DMS, 0: Null, 1: Shared lock, 2: Exclusive lock @@ -527,6 +547,12 @@ typedef struct st_dcs_batch_buf { unsigned int max_count; } dcs_batch_buf_t; +typedef enum en_dms_inst_behavior { + DMS_INST_BEHAVIOR_IN_IDLE = 0, + DMS_INST_BEHAVIOR_IN_REFORM, + DMS_INST_BEHAVIOR_IN_BACKUP, +} dms_inst_behavior_t; + typedef int(*dms_get_list_stable)(void *db_handle, unsigned long long *list_stable, unsigned char *reformer_id); typedef int(*dms_save_list_stable)(void *db_handle, unsigned long long list_stable, unsigned char reformer_id, unsigned long long list_in, unsigned int save_ctrl); @@ -605,9 +631,8 @@ typedef char *(*dms_mem_alloc)(void *context, unsigned int size); typedef void(*dms_mem_free)(void *context, void *ptr); typedef void(*dms_mem_reset)(void *context); // The maximum length of output_msg is 128 bytes. -typedef int (*dms_process_broadcast)(void *db_handle, char *data, unsigned int len, char *output_msg, - unsigned int *output_msg_len); -typedef int (*dms_process_broadcast_ack)(void *db_handle, char *data, unsigned int len); +typedef int (*dms_process_broadcast)(void *db_handle, dms_broadcast_context_t *broad_ctx); +typedef int (*dms_process_broadcast_ack)(void *db_handle, dms_broadcast_context_t *broad_ctx); typedef int(*dms_get_txn_info)(void *db_handle, unsigned long long xid, unsigned char is_scan, dms_txn_info_t *txn_info); typedef int(*dms_get_opengauss_xid_csn)(void *db_handle, dms_opengauss_xid_csn_t *csn_req, @@ -637,12 +662,14 @@ typedef int (*dms_drc_buf_res_rebuild_parallel)(void *db_handle, unsigned char t typedef int(*dms_ctl_rcy_clean_parallel_t)(void *db_handle, unsigned char thread_index, unsigned char thread_num); typedef unsigned char(*dms_ckpt_session)(void *db_handle); typedef void (*dms_check_if_build_complete)(void *db_handle, unsigned int *build_complete); +typedef void (*dms_check_if_restore_recover)(void *db_handle, unsigned int *rst_recover); typedef int (*dms_db_is_primary)(void *db_handle); typedef void (*dms_set_switchover_result)(void *db_handle, int result); typedef void (*dms_set_db_role)(void *db_handle, unsigned char is_primary); typedef int (*dms_mount_to_recovery)(void *db_handle, unsigned int *has_offline); typedef int(*dms_get_open_status)(void *db_handle); typedef void (*dms_reform_set_dms_role)(void *db_handle, unsigned int reformer_id); +typedef void (*dms_reset_user)(void *db_handle, unsigned long long list_in); // for openGauss typedef void (*dms_thread_init_t)(unsigned char need_startup, char **reg_data); @@ -682,6 +709,7 @@ typedef int (*dms_get_enable_checksum)(void *db_handle); typedef unsigned int (*dms_calc_page_checksum)(void *db_handle, dms_buf_ctrl_t *ctrl, unsigned int page_size); typedef int (*dms_verify_page_checksum)(void *db_handle, dms_buf_ctrl_t *ctrl, unsigned int page_size, int cks); typedef int (*dms_update_node_oldest_xmin)(void *db_handle, unsigned char inst_id, unsigned long long oldest_xmin); +typedef void (*dms_set_inst_behavior)(void *db_handle, dms_inst_behavior_t inst_behavior); typedef struct st_dms_callback { // used in reform @@ -711,6 +739,8 @@ typedef struct st_dms_callback { dms_drc_buf_res_rebuild_parallel dms_reform_rebuild_parallel; dms_ctl_rcy_clean_parallel_t dms_ctl_rcy_clean_parallel; dms_check_if_build_complete check_if_build_complete; + dms_check_if_restore_recover check_if_restore_recover; + dms_reset_user reset_user; // used in reform for opengauss dms_thread_init_t dms_thread_init; @@ -821,6 +851,9 @@ typedef struct st_dms_callback { dms_calc_page_checksum calc_page_checksum; dms_verify_page_checksum verify_page_checksum; dms_update_node_oldest_xmin update_node_oldest_xmin; + + //for shared storage backup + dms_set_inst_behavior set_inst_behavior; } dms_callback_t; typedef struct st_dms_instance_net_addr { @@ -892,11 +925,19 @@ typedef enum en_dms_info_id { DMS_INFO_REFORM_LAST = 1, } dms_info_id_e; +typedef enum st_protocol_version { + PROTO_VER_0 = 0, // invalid version + PROTO_VER_1 = 1, // first version +} protocol_version_e; + +#define INVALID_PROTO_VER PROTO_VER_0 +#define SW_PROTO_VER PROTO_VER_1 + #define DMS_LOCAL_MAJOR_VER_WEIGHT 1000000 #define DMS_LOCAL_MINOR_VER_WEIGHT 1000 #define DMS_LOCAL_MAJOR_VERSION 0 #define DMS_LOCAL_MINOR_VERSION 0 -#define DMS_LOCAL_VERSION 84 +#define DMS_LOCAL_VERSION 88 #ifdef __cplusplus } From 04726e9fa8d2fa10fd7c69f06ea48ffb204828a6 Mon Sep 17 00:00:00 2001 From: justbk <249396768@qq.com> Date: Thu, 10 Aug 2023 17:03:10 +0800 Subject: [PATCH 155/304] support new db_time time record. --- src/bin/gs_guc/cluster_guc.conf | 1 + src/common/backend/catalog/builtin_funcs.ini | 8 +- .../backend/catalog/performance_views.sql | 103 +- src/common/backend/catalog/system_views.sql | 35 +- src/common/backend/libpq/be-secure.cpp | 2 +- src/common/backend/utils/adt/pgstatfuncs.cpp | 15 +- src/common/backend/utils/init/globals.cpp | 2 +- src/common/backend/utils/misc/guc.cpp | 13 + src/common/pl/plpgsql/src/pl_handler.cpp | 6 +- .../instruments/statement/instr_statement.cpp | 12 +- .../unique_sql/instr_unique_sql.cpp | 23 +- src/gausskernel/process/postmaster/Makefile | 2 +- .../process/postmaster/og_record_time.cpp | 698 ++++++++++ src/gausskernel/process/postmaster/pgstat.cpp | 103 +- src/gausskernel/process/tcop/postgres.cpp | 52 +- src/gausskernel/process/tcop/pquery.cpp | 2 + .../process/threadpool/knl_session.cpp | 2 + .../runtime/executor/lightProxy.cpp | 1 + src/gausskernel/storage/ipc/ipc.cpp | 3 +- .../rollback-post_catalog_maindb_92_906.sql | 955 ++++++++++++++ .../rollback_catalog_maindb_92_906.sql | 22 + .../rollback-post_catalog_otherdb_92_906.sql | 956 ++++++++++++++ .../rollback_catalog_otherdb_92_906.sql | 22 + .../upgrade-post_catalog_maindb_92_906.sql | 1127 +++++++++++++++++ .../upgrade_catalog_maindb_92_906.sql | 22 + .../upgrade-post_catalog_otherdb_92_906.sql | 1127 +++++++++++++++++ .../upgrade_catalog_otherdb_92_906.sql | 22 + .../knl/knl_guc/knl_session_attr_common.h | 1 + src/include/knl/knl_session.h | 3 + src/include/knl/knl_thread.h | 1 + src/include/og_record_time.h | 512 ++++++++ src/include/og_record_time_rely.h | 33 + src/include/pgstat.h | 112 +- .../regress/output/recovery_2pc_tools.source | 1 + 34 files changed, 5818 insertions(+), 181 deletions(-) create mode 100644 src/gausskernel/process/postmaster/og_record_time.cpp create mode 100644 src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback_catalog_maindb_92_906.sql create mode 100644 src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback_catalog_otherdb_92_906.sql create mode 100644 src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade_catalog_maindb_92_906.sql create mode 100644 src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade_catalog_otherdb_92_906.sql create mode 100644 src/include/og_record_time.h create mode 100644 src/include/og_record_time_rely.h diff --git a/src/bin/gs_guc/cluster_guc.conf b/src/bin/gs_guc/cluster_guc.conf index 922395615..321353905 100755 --- a/src/bin/gs_guc/cluster_guc.conf +++ b/src/bin/gs_guc/cluster_guc.conf @@ -755,6 +755,7 @@ ignore_standby_lsn_window|int|0,2147483647|ms|NULL| ignore_feedback_xmin_window|int|0,2147483647|ms|NULL| ss_enable_bcast_snapshot|bool|0,0|NULL|NULL| subscription_conflict_resolution|enum|error,apply_remote,keep_local|NULL|NULL| +time_record_level|int|0,10|NULL|NULL| [cmserver] log_dir|string|0,0|NULL|NULL| log_file_size|int|0,2047|MB|NULL| diff --git a/src/common/backend/catalog/builtin_funcs.ini b/src/common/backend/catalog/builtin_funcs.ini index 2ea8be3fa..0c59329f6 100755 --- a/src/common/backend/catalog/builtin_funcs.ini +++ b/src/common/backend/catalog/builtin_funcs.ini @@ -3179,7 +3179,7 @@ ), AddFuncGroup( "get_instr_unique_sql", 1, - AddBuiltinFunc(_0(5702), _1("get_instr_unique_sql"), _2(0), _3(false), _4(true), _5(get_instr_unique_sql), _6(2249), _7(PG_CATALOG_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(1000), _12(0), _13(0), _14(false), _15(false), _16(false), _17(false), _18('s'), _19(0), _20(0), _21(44, 19, 23, 19, 26, 20, 25, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 25, 25, 25, 25, 1184, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20), _22(44, 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o','o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o'), _23(44, "node_name", "node_id", "user_name", "user_id", "unique_sql_id", "query", "n_calls", "min_elapse_time", "max_elapse_time", "total_elapse_time", "n_returned_rows", "n_tuples_fetched", "n_tuples_returned", "n_tuples_inserted", "n_tuples_updated", "n_tuples_deleted", "n_blocks_fetched", "n_blocks_hit", "n_soft_parse", "n_hard_parse", "db_time", "cpu_time", "execution_time", "parse_time", "plan_time", "rewrite_time", "pl_execution_time", "pl_compilation_time", "data_io_time", "net_send_info", "net_recv_info", "net_stream_send_info", "net_stream_recv_info", "last_updated", "sort_count", "sort_time", "sort_mem_used", "sort_spill_count", "sort_spill_size", "hash_count", "hash_time", "hash_mem_used", "hash_spill_count", "hash_spill_size"), _24(NULL), _25("get_instr_unique_sql"), _26(NULL), _27(NULL), _28(NULL), _29(0), _30(false), _31(NULL), _32(false), _33(NULL), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)) + AddBuiltinFunc(_0(5702), _1("get_instr_unique_sql"), _2(0), _3(false), _4(true), _5(get_instr_unique_sql), _6(2249), _7(PG_CATALOG_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(1000), _12(0), _13(0), _14(false), _15(false), _16(false), _17(false), _18('s'), _19(0), _20(0), _21(60, 19, 23, 19, 26, 20, 25, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 25, 25, 25, 25, 1184, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20), _22(60, 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o','o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o'), _23(60, "node_name", "node_id", "user_name", "user_id", "unique_sql_id", "query", "n_calls", "min_elapse_time", "max_elapse_time", "total_elapse_time", "n_returned_rows", "n_tuples_fetched", "n_tuples_returned", "n_tuples_inserted", "n_tuples_updated", "n_tuples_deleted", "n_blocks_fetched", "n_blocks_hit", "n_soft_parse", "n_hard_parse", "db_time", "cpu_time", "execution_time", "parse_time", "plan_time", "rewrite_time", "pl_execution_time", "pl_compilation_time", "data_io_time", "net_send_info", "net_recv_info", "net_stream_send_info", "net_stream_recv_info", "last_updated", "sort_count", "sort_time", "sort_mem_used", "sort_spill_count", "sort_spill_size", "hash_count", "hash_time", "hash_mem_used", "hash_spill_count", "hash_spill_size", "net_send_time", "srt1_q", "srt2_simple_query", "srt3_analyze_rewrite", "srt4_plan_query", "srt5_light_query", "srt6_p", "srt7_b", "srt8_e", "srt9_d", "srt10_s", "srt11_c", "srt12_u", "srt13_before_query", "srt14_after_query","rtt_unknown"), _24(NULL), _25("get_instr_unique_sql"), _26(NULL), _27(NULL), _28(NULL), _29(0), _30(false), _31(NULL), _32(false), _33(NULL), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)) ), AddFuncGroup( "get_instr_user_login", 1, @@ -9105,7 +9105,7 @@ ), AddFuncGroup( "pg_stat_get_wlm_session_info", 1, - AddBuiltinFunc(_0(5002), _1("pg_stat_get_wlm_session_info"), _2(1), _3(false), _4(true), _5(pg_stat_get_wlm_session_info), _6(2249), _7(PG_CATALOG_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(100), _12(0), _13(0), _14(false), _15(false), _16(false), _17(false), _18('s'), _19(0), _20(1, 26), _21(87, 26, 25, 25, 25, 25, 25, 869, 25, 23, 25, 20, 1184, 1184, 20, 20, 25, 25, 25, 25, 23, 23, 23, 23, 23, 25, 23, 23, 23, 23, 20, 20, 20, 23, 20, 20, 20, 23, 23, 23, 23, 23, 25, 20, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 25, 25, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20), _22(87, 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o'), _23(87, "datid", "dbname", "schemaname", "nodename", "username", "application_name", "client_addr", "client_hostname", "client_port", "query_band", "block_time", "start_time", "finish_time", "duration", "estimate_total_time", "status", "abort_info", "resource_pool", "control_group", "estimate_memory", "min_peak_memory", "max_peak_memory", "average_peak_memory", "memory_skew_percent", "spill_info", "min_spill_size", "max_spill_size", "average_spill_size", "spill_skew_percent", "min_dn_time", "max_dn_time", "average_dn_time", "dntime_skew_percent", "min_cpu_time", "max_cpu_time", "total_cpu_time", "cpu_skew_percent", "min_peak_iops", "max_peak_iops", "average_peak_iops", "iops_skew_percent", "warning", "queryid", "query", "query_plan", "node_group", "cpu_top1_node_name", "cpu_top2_node_name", "cpu_top3_node_name", "cpu_top4_node_name", "cpu_top5_node_name", "mem_top1_node_name", "mem_top2_node_name", "mem_top3_node_name", "mem_top4_node_name", "mem_top5_node_name", "cpu_top1_value", "cpu_top2_value", "cpu_top3_value", "cpu_top4_value", "cpu_top5_value", "mem_top1_value", "mem_top2_value", "mem_top3_value", "mem_top4_value", "mem_top5_value", "top_mem_dn", "top_cpu_dn", "n_returned_rows", "n_tuples_fetched", "n_tuples_returned", "n_tuples_inserted", "n_tuples_updated", "n_tuples_deleted", "n_blocks_fetched", "n_blocks_hit", "db_time", "cpu_time", "execution_time", "parse_time", "plan_time", "rewrite_time", "pl_execution_time", "pl_compilation_time", "net_send_time", "data_io_time", "is_slow_query"), _24(NULL), _25("pg_stat_get_wlm_session_info"), _26(NULL), _27(NULL), _28(NULL), _29(0), _30(false), _31(NULL), _32(false), _33(NULL), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)) + AddBuiltinFunc(_0(5002), _1("pg_stat_get_wlm_session_info"), _2(1), _3(false), _4(true), _5(pg_stat_get_wlm_session_info), _6(2249), _7(PG_CATALOG_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(100), _12(0), _13(0), _14(false), _15(false), _16(false), _17(false), _18('s'), _19(0), _20(1, 26), _21(102, 26, 25, 25, 25, 25, 25, 869, 25, 23, 25, 20, 1184, 1184, 20, 20, 25, 25, 25, 25, 23, 23, 23, 23, 23, 25, 23, 23, 23, 23, 20, 20, 20, 23, 20, 20, 20, 23, 23, 23, 23, 23, 25, 20, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 25, 25, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20), _22(102, 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o'), _23(102, "datid", "dbname", "schemaname", "nodename", "username", "application_name", "client_addr", "client_hostname", "client_port", "query_band", "block_time", "start_time", "finish_time", "duration", "estimate_total_time", "status", "abort_info", "resource_pool", "control_group", "estimate_memory", "min_peak_memory", "max_peak_memory", "average_peak_memory", "memory_skew_percent", "spill_info", "min_spill_size", "max_spill_size", "average_spill_size", "spill_skew_percent", "min_dn_time", "max_dn_time", "average_dn_time", "dntime_skew_percent", "min_cpu_time", "max_cpu_time", "total_cpu_time", "cpu_skew_percent", "min_peak_iops", "max_peak_iops", "average_peak_iops", "iops_skew_percent", "warning", "queryid", "query", "query_plan", "node_group", "cpu_top1_node_name", "cpu_top2_node_name", "cpu_top3_node_name", "cpu_top4_node_name", "cpu_top5_node_name", "mem_top1_node_name", "mem_top2_node_name", "mem_top3_node_name", "mem_top4_node_name", "mem_top5_node_name", "cpu_top1_value", "cpu_top2_value", "cpu_top3_value", "cpu_top4_value", "cpu_top5_value", "mem_top1_value", "mem_top2_value", "mem_top3_value", "mem_top4_value", "mem_top5_value", "top_mem_dn", "top_cpu_dn", "n_returned_rows", "n_tuples_fetched", "n_tuples_returned", "n_tuples_inserted", "n_tuples_updated", "n_tuples_deleted", "n_blocks_fetched", "n_blocks_hit", "db_time", "cpu_time", "execution_time", "parse_time", "plan_time", "rewrite_time", "pl_execution_time", "pl_compilation_time", "net_send_time", "data_io_time", "is_slow_query", "srt1_q", "srt2_simple_query", "srt3_analyze_rewrite", "srt4_plan_query", "srt5_light_query", "srt6_p", "srt7_b", "srt8_e", "srt9_d", "srt10_s", "srt11_c", "srt12_u", "srt13_before_query", "srt14_after_query","rtt_unknown"), _24(NULL), _25("pg_stat_get_wlm_session_info"), _26(NULL), _27(NULL), _28(NULL), _29(0), _30(false), _31(NULL), _32(false), _33(NULL), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)) ), AddFuncGroup( "pg_stat_get_wlm_session_info_internal", 1, @@ -10997,8 +10997,8 @@ AddFuncGroup( ), AddFuncGroup( "standby_statement_history", 2, - AddBuiltinFunc(_0(3118), _1("standby_statement_history"), _2(1), _3(false), _4(true), _5(standby_statement_history_1v), _6(2249), _7(PG_DBEPERF_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(10000), _12(0), _13(0), _14(false), _15(false), _16(false), _17(false), _18('v'), _19(0), _20(1, 16), _21(54, 16, 19, 19, 23, 19, 25, 25, 23, 20, 20, 25, 1184, 1184, 20, 20, 20, 20, 20, 20, 25, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 25, 25, 25, 25, 20, 20, 20, 20, 20, 20, 20, 20, 20, 17, 16, 25, 25), _22(54, 'i', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o'), _23(54, "only_slow", "db_name", "schema_name", "origin_node", "user_name", "application_name", "client_addr", "client_port", "unique_query_id", "debug_query_id", "query", "start_time", "finish_time", "slow_sql_threshold", "transaction_id", "thread_id", "session_id", "n_soft_parse", "n_hard_parse", "query_plan", "n_returned_rows", "n_tuples_fetched", "n_tuples_returned", "n_tuples_inserted", "n_tuples_updated", "n_tuples_deleted", "n_blocks_fetched", "n_blocks_hit", "db_time", "cpu_time", "execution_time", "parse_time", "plan_time", "rewrite_time", "pl_execution_time", "pl_compilation_time", "data_io_time", "net_send_info", "net_recv_info", "net_stream_send_info", "net_stream_recv_info", "lock_count", "lock_time", "lock_wait_count", "lock_wait_time", "lock_max_count", "lwlock_count", "lwlock_wait_count", "lwlock_time", "lwlock_wait_time", "details", "is_slow_sql", "trace_id", "advise"),_24(NULL), _25("standby_statement_history_1v"), _26(NULL), _27(NULL), _28(NULL), _29(0), _30(false), _31(NULL), _32(false), _33(NULL), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)), - AddBuiltinFunc(_0(3119), _1("standby_statement_history"), _2(1), _3(false), _4(true), _5(standby_statement_history), _6(2249), _7(PG_DBEPERF_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(10000), _12(1185), _13(0), _14(false), _15(false), _16(false), _17(false), _18('v'), _19(0), _20(2, 16, 1185), _21(55, 16, 1185, 19, 19, 23, 19, 25, 25, 23, 20, 20, 25, 1184, 1184, 20, 20, 20, 20, 20, 20, 25, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 25, 25, 25, 25, 20, 20, 20, 20, 20, 20, 20, 20, 20, 17, 16, 25, 25), _22(55, 'i', 'v', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o'), _23(55, "only_slow", "finish_time", "db_name", "schema_name", "origin_node", "user_name", "application_name", "client_addr", "client_port", "unique_query_id", "debug_query_id", "query", "start_time", "finish_time", "slow_sql_threshold", "transaction_id", "thread_id", "session_id", "n_soft_parse", "n_hard_parse", "query_plan", "n_returned_rows", "n_tuples_fetched", "n_tuples_returned", "n_tuples_inserted", "n_tuples_updated", "n_tuples_deleted", "n_blocks_fetched", "n_blocks_hit", "db_time", "cpu_time", "execution_time", "parse_time", "plan_time", "rewrite_time", "pl_execution_time", "pl_compilation_time", "data_io_time", "net_send_info", "net_recv_info", "net_stream_send_info", "net_stream_recv_info", "lock_count", "lock_time", "lock_wait_count", "lock_wait_time", "lock_max_count", "lwlock_count", "lwlock_wait_count", "lwlock_time", "lwlock_wait_time", "details", "is_slow_sql", "trace_id", "advise"),_24(NULL), _25("standby_statement_history"), _26(NULL), _27(NULL), _28(NULL), _29(0), _30(false), _31(NULL), _32(false), _33(NULL), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)) + AddBuiltinFunc(_0(3118), _1("standby_statement_history"), _2(1), _3(false), _4(true), _5(standby_statement_history_1v), _6(2249), _7(PG_DBEPERF_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(10000), _12(0), _13(0), _14(false), _15(false), _16(false), _17(false), _18('v'), _19(0), _20(1, 16), _21(70, 16, 19, 19, 23, 19, 25, 25, 23, 20, 20, 25, 1184, 1184, 20, 20, 20, 20, 20, 20, 25, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 25, 25, 25, 25, 20, 20, 20, 20, 20, 20, 20, 20, 20, 17, 16, 25, 25, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20), _22(70, 'i', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o'), _23(70, "only_slow", "db_name", "schema_name", "origin_node", "user_name", "application_name", "client_addr", "client_port", "unique_query_id", "debug_query_id", "query", "start_time", "finish_time", "slow_sql_threshold", "transaction_id", "thread_id", "session_id", "n_soft_parse", "n_hard_parse", "query_plan", "n_returned_rows", "n_tuples_fetched", "n_tuples_returned", "n_tuples_inserted", "n_tuples_updated", "n_tuples_deleted", "n_blocks_fetched", "n_blocks_hit", "db_time", "cpu_time", "execution_time", "parse_time", "plan_time", "rewrite_time", "pl_execution_time", "pl_compilation_time", "data_io_time", "net_send_info", "net_recv_info", "net_stream_send_info", "net_stream_recv_info", "lock_count", "lock_time", "lock_wait_count", "lock_wait_time", "lock_max_count", "lwlock_count", "lwlock_wait_count", "lwlock_time", "lwlock_wait_time", "details", "is_slow_sql", "trace_id", "advise", "net_send_time","srt1_q", "srt2_simple_query", "srt3_analyze_rewrite", "srt4_plan_query", "srt5_light_query", "srt6_p", "srt7_b", "srt8_e", "srt9_d", "srt10_s", "srt11_c", "srt12_u", "srt13_before_query", "srt14_after_query","rtt_unknown"),_24(NULL), _25("standby_statement_history_1v"), _26(NULL), _27(NULL), _28(NULL), _29(0), _30(false), _31(NULL), _32(false), _33(NULL), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)), + AddBuiltinFunc(_0(3119), _1("standby_statement_history"), _2(1), _3(false), _4(true), _5(standby_statement_history), _6(2249), _7(PG_DBEPERF_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(10000), _12(1185), _13(0), _14(false), _15(false), _16(false), _17(false), _18('v'), _19(0), _20(2, 16, 1185), _21(71, 16, 1185, 19, 19, 23, 19, 25, 25, 23, 20, 20, 25, 1184, 1184, 20, 20, 20, 20, 20, 20, 25, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 25, 25, 25, 25, 20, 20, 20, 20, 20, 20, 20, 20, 20, 17, 16, 25, 25, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20), _22(71, 'i', 'v', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o'), _23(71, "only_slow", "finish_time", "db_name", "schema_name", "origin_node", "user_name", "application_name", "client_addr", "client_port", "unique_query_id", "debug_query_id", "query", "start_time", "finish_time", "slow_sql_threshold", "transaction_id", "thread_id", "session_id", "n_soft_parse", "n_hard_parse", "query_plan", "n_returned_rows", "n_tuples_fetched", "n_tuples_returned", "n_tuples_inserted", "n_tuples_updated", "n_tuples_deleted", "n_blocks_fetched", "n_blocks_hit", "db_time", "cpu_time", "execution_time", "parse_time", "plan_time", "rewrite_time", "pl_execution_time", "pl_compilation_time", "data_io_time", "net_send_info", "net_recv_info", "net_stream_send_info", "net_stream_recv_info", "lock_count", "lock_time", "lock_wait_count", "lock_wait_time", "lock_max_count", "lwlock_count", "lwlock_wait_count", "lwlock_time", "lwlock_wait_time", "details", "is_slow_sql", "trace_id", "advise", "net_send_time","srt1_q", "srt2_simple_query", "srt3_analyze_rewrite", "srt4_plan_query", "srt5_light_query", "srt6_p", "srt7_b", "srt8_e", "srt9_d", "srt10_s", "srt11_c", "srt12_u", "srt13_before_query", "srt14_after_query","rtt_unknown"),_24(NULL), _25("standby_statement_history"), _26(NULL), _27(NULL), _28(NULL), _29(0), _30(false), _31(NULL), _32(false), _33(NULL), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)) ), AddFuncGroup( "statement_detail_decode", 1, diff --git a/src/common/backend/catalog/performance_views.sql b/src/common/backend/catalog/performance_views.sql index 0af3a1e02..4635a88af 100644 --- a/src/common/backend/catalog/performance_views.sql +++ b/src/common/backend/catalog/performance_views.sql @@ -3994,7 +3994,24 @@ CREATE OR REPLACE FUNCTION DBE_PERF.get_global_full_sql_by_timestamp OUT details bytea, OUT is_slow_sql bool, OUT trace_id text, - OUT advise text) + OUT advise text, + OUT net_send_time bigint, + OUT srt1_q bigint, + OUT srt2_simple_query bigint, + OUT srt3_analyze_rewrite bigint, + OUT srt4_plan_query bigint, + OUT srt5_light_query bigint, + OUT srt6_p bigint, + OUT srt7_b bigint, + OUT srt8_e bigint, + OUT srt9_d bigint, + OUT srt10_s bigint, + OUT srt11_c bigint, + OUT srt12_u bigint, + OUT srt13_before_query bigint, + OUT srt14_after_query bigint, + OUT rtt_unknown bigint + ) RETURNS setof record AS $$ DECLARE @@ -4063,6 +4080,22 @@ CREATE OR REPLACE FUNCTION DBE_PERF.get_global_full_sql_by_timestamp is_slow_sql := row_data.is_slow_sql; trace_id := row_data.trace_id; advise := row_data.advise; + net_send_time =row_data.net_send_time; + srt1_q := row_data.srt1_q; + srt2_simple_query := row_data.srt2_simple_query; + srt3_analyze_rewrite := row_data.srt3_analyze_rewrite; + srt4_plan_query := row_data.srt4_plan_query; + srt5_light_query := row_data.srt5_light_query; + srt6_p := row_data.srt6_p; + srt7_b := row_data.srt7_b; + srt8_e := row_data.srt8_e; + srt9_d := row_data.srt9_d; + srt10_s := row_data.srt10_s; + srt11_c := row_data.srt11_c; + srt12_u := row_data.srt12_u; + srt13_before_query := row_data.srt13_before_query; + srt14_after_query := row_data.srt14_after_query; + rtt_unknown := row_data.rtt_unknown; return next; END LOOP; END LOOP; @@ -4126,7 +4159,23 @@ CREATE OR REPLACE FUNCTION DBE_PERF.get_global_slow_sql_by_timestamp OUT details bytea, OUT is_slow_sql bool, OUT trace_id text, - OUT advise text) + OUT advise text, + OUT net_send_time bigint, + OUT srt1_q bigint, + OUT srt2_simple_query bigint, + OUT srt3_analyze_rewrite bigint, + OUT srt4_plan_query bigint, + OUT srt5_light_query bigint, + OUT srt6_p bigint, + OUT srt7_b bigint, + OUT srt8_e bigint, + OUT srt9_d bigint, + OUT srt10_s bigint, + OUT srt11_c bigint, + OUT srt12_u bigint, + OUT srt13_before_query bigint, + OUT srt14_after_query bigint, + OUT rtt_unknown bigint) RETURNS setof record AS $$ DECLARE @@ -4195,6 +4244,22 @@ CREATE OR REPLACE FUNCTION DBE_PERF.get_global_slow_sql_by_timestamp is_slow_sql := row_data.is_slow_sql; trace_id := row_data.trace_id; advise := row_data.advise; + net_send_time =row_data.net_send_time; + srt1_q := row_data.srt1_q; + srt2_simple_query := row_data.srt2_simple_query; + srt3_analyze_rewrite := row_data.srt3_analyze_rewrite; + srt4_plan_query := row_data.srt4_plan_query; + srt5_light_query := row_data.srt5_light_query; + srt6_p := row_data.srt6_p; + srt7_b := row_data.srt7_b; + srt8_e := row_data.srt8_e; + srt9_d := row_data.srt9_d; + srt10_s := row_data.srt10_s; + srt11_c := row_data.srt11_c; + srt12_u := row_data.srt12_u; + srt13_before_query := row_data.srt13_before_query; + srt14_after_query := row_data.srt14_after_query; + rtt_unknown := row_data.rtt_unknown; return next; END LOOP; END LOOP; @@ -4553,7 +4618,22 @@ SELECT S.pl_execution_time, S.pl_compilation_time, S.net_send_time, - S.data_io_time + S.data_io_time, + S.srt1_q, + S.srt2_simple_query, + S.srt3_analyze_rewrite, + S.srt4_plan_query, + S.srt5_light_query, + S.srt6_p, + S.srt7_b, + S.srt8_e, + S.srt9_d, + S.srt10_s, + S.srt11_c, + S.srt12_u, + S.srt13_before_query, + S.srt14_after_query, + S.rtt_unknown FROM gs_wlm_session_query_info_all S where S.is_slow_query = 1; CREATE VIEW dbe_perf.gs_slow_query_history AS @@ -4585,7 +4665,22 @@ SELECT S.pl_execution_time, S.pl_compilation_time, S.net_send_time, - S.data_io_time + S.data_io_time, + S.srt1_q, + S.srt2_simple_query, + S.srt3_analyze_rewrite, + S.srt4_plan_query, + S.srt5_light_query, + S.srt6_p, + S.srt7_b, + S.srt8_e, + S.srt9_d, + S.srt10_s, + S.srt11_c, + S.srt12_u, + S.srt13_before_query, + S.srt14_after_query, + S.rtt_unknown FROM pg_catalog.pg_stat_get_wlm_session_info(0) S where S.is_slow_query = 1; CREATE OR REPLACE FUNCTION dbe_perf.global_slow_query_history diff --git a/src/common/backend/catalog/system_views.sql b/src/common/backend/catalog/system_views.sql index 41d9f8cc1..965933cdb 100644 --- a/src/common/backend/catalog/system_views.sql +++ b/src/common/backend/catalog/system_views.sql @@ -1386,7 +1386,22 @@ create table gs_wlm_session_query_info_all pl_compilation_time bigint, net_send_time bigint, data_io_time bigint, - is_slow_query bigint + is_slow_query bigint, + srt1_q bigint, + srt2_simple_query bigint, + srt3_analyze_rewrite bigint, + srt4_plan_query bigint, + srt5_light_query bigint, + srt6_p bigint, + srt7_b bigint, + srt8_e bigint, + srt9_d bigint, + srt10_s bigint, + srt11_c bigint, + srt12_u bigint, + srt13_before_query bigint, + srt14_after_query bigint, + rtt_unknown bigint ); CREATE VIEW gs_wlm_session_info_all AS @@ -3487,7 +3502,23 @@ CREATE unlogged table statement_history( details bytea, is_slow_sql bool, trace_id text, - advise text + advise text, + net_send_time bigint, + srt1_q bigint, + srt2_simple_query bigint, + srt3_analyze_rewrite bigint, + srt4_plan_query bigint, + srt5_light_query bigint, + srt6_p bigint, + srt7_b bigint, + srt8_e bigint, + srt9_d bigint, + srt10_s bigint, + srt11_c bigint, + srt12_u bigint, + srt13_before_query bigint, + srt14_after_query bigint, + rtt_unknown bigint ); REVOKE ALL on table pg_catalog.statement_history FROM public; create index statement_history_time_idx on pg_catalog.statement_history USING btree (start_time, is_slow_sql); diff --git a/src/common/backend/libpq/be-secure.cpp b/src/common/backend/libpq/be-secure.cpp index 80bbfb7a7..9504fa20c 100644 --- a/src/common/backend/libpq/be-secure.cpp +++ b/src/common/backend/libpq/be-secure.cpp @@ -572,7 +572,7 @@ ssize_t old_secure_write(Port* port, void* ptr, size_t len) /* CommProxy Interface Support */ n = comm_send(port->sock, ptr, len, 0); PGSTAT_END_TIME_RECORD(NET_SEND_TIME); - END_NET_SEND_INFO(n); + END_NET_SEND_INFO_DUPLICATE(n); /* for log printing, send message */ IPC_PERFORMANCE_LOG_COLLECT(port->msgLog, ptr, n, port->remote_hostname, NULL, SECURE_WRITE); diff --git a/src/common/backend/utils/adt/pgstatfuncs.cpp b/src/common/backend/utils/adt/pgstatfuncs.cpp index 01e1d7966..df34a6015 100644 --- a/src/common/backend/utils/adt/pgstatfuncs.cpp +++ b/src/common/backend/utils/adt/pgstatfuncs.cpp @@ -6103,7 +6103,7 @@ Datum pg_stat_get_wlm_session_info(PG_FUNCTION_ARGS) int WLM_SESSION_INFO_ATTRNUM = 0; if (t_thrd.proc->workingVersionNum >= SLOW_QUERY_VERSION) - WLM_SESSION_INFO_ATTRNUM = 87; + WLM_SESSION_INFO_ATTRNUM = 77 + TOTAL_TIME_INFO_TYPES; else WLM_SESSION_INFO_ATTRNUM = 68; @@ -6201,10 +6201,13 @@ Datum pg_stat_get_wlm_session_info(PG_FUNCTION_ARGS) TupleDescInitEntry(tupdesc, (AttrNumber)++i, "n_tuples_deleted", INT8OID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber)++i, "t_blocks_fetched", INT8OID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber)++i, "t_blocks_hit", INT8OID, -1, 0); - for (num = 0; num < TOTAL_TIME_INFO_TYPES; num++) { + for (num = 0; num < TOTAL_TIME_INFO_TYPES_P1; num++) { TupleDescInitEntry(tupdesc, (AttrNumber)++i, TimeInfoTypeName[num], INT8OID, -1, 0); } TupleDescInitEntry(tupdesc, (AttrNumber)++i, "is_slow_query", INT8OID, -1, 0); + for (num = TOTAL_TIME_INFO_TYPES_P1; num < TOTAL_TIME_INFO_TYPES; num++) { + TupleDescInitEntry(tupdesc, (AttrNumber)++i, TimeInfoTypeName[num], INT8OID, -1, 0); + } } funcctx->tuple_desc = BlessTupleDesc(tupdesc); funcctx->user_fctx = WLMGetSessionInfo(&qid, removed, &num); @@ -6439,11 +6442,15 @@ Datum pg_stat_get_wlm_session_info(PG_FUNCTION_ARGS) values[++i] = Int64GetDatum(detail->gendata.slowQueryInfo.current_table_counter->t_tuples_deleted); values[++i] = Int64GetDatum(detail->gendata.slowQueryInfo.current_table_counter->t_blocks_fetched); values[++i] = Int64GetDatum(detail->gendata.slowQueryInfo.current_table_counter->t_blocks_hit); - /* time Info */ - for (num = 0; num < TOTAL_TIME_INFO_TYPES; num++) { + /* time Info p1*/ + for (num = 0; num < TOTAL_TIME_INFO_TYPES_P1; num++) { values[++i] = Int64GetDatum(detail->gendata.slowQueryInfo.localTimeInfoArray[num]); } values[++i] = Int64GetDatum(0); + /* time Info */ + for (num = TOTAL_TIME_INFO_TYPES_P1; num < TOTAL_TIME_INFO_TYPES; num++) { + values[++i] = Int64GetDatum(detail->gendata.slowQueryInfo.localTimeInfoArray[num]); + } } tuple = heap_form_tuple(funcctx->tuple_desc, values, nulls); result = HeapTupleGetDatum(tuple); diff --git a/src/common/backend/utils/init/globals.cpp b/src/common/backend/utils/init/globals.cpp index f805918a8..139cb5b62 100644 --- a/src/common/backend/utils/init/globals.cpp +++ b/src/common/backend/utils/init/globals.cpp @@ -75,7 +75,7 @@ bool will_shutdown = false; * NEXT | 92899 | ? | ? * ********************************************/ -const uint32 GRAND_VERSION_NUM = 92905; +const uint32 GRAND_VERSION_NUM = 92906; /******************************************** * 2.VERSION NUM FOR EACH FEATURE diff --git a/src/common/backend/utils/misc/guc.cpp b/src/common/backend/utils/misc/guc.cpp index 35469e94f..87f65f653 100755 --- a/src/common/backend/utils/misc/guc.cpp +++ b/src/common/backend/utils/misc/guc.cpp @@ -3033,6 +3033,19 @@ static void InitConfigureNamesInt() NULL, NULL, NULL}, + {{"time_record_level", + PGC_USERSET, + NODE_SINGLENODE, + STATS_COLLECTOR, + gettext_noop("Set time record level."), + NULL}, + &u_sess->attr.attr_common.time_record_level, + 0, + 0, + 10, + NULL, + NULL, + NULL}, /* End-of-list marker */ {{NULL, (GucContext)0, diff --git a/src/common/pl/plpgsql/src/pl_handler.cpp b/src/common/pl/plpgsql/src/pl_handler.cpp index edc8d9b14..3d1967f08 100755 --- a/src/common/pl/plpgsql/src/pl_handler.cpp +++ b/src/common/pl/plpgsql/src/pl_handler.cpp @@ -699,8 +699,7 @@ Datum plpgsql_call_handler(PG_FUNCTION_ARGS) int save_sec_context = 0; Oid cast_owner = InvalidOid; bool has_switch = false; - // PGSTAT_INIT_PLSQL_TIME_RECORD - int64 startTime = 0; + PGSTAT_INIT_TIME_RECORD(); bool needRecord = false; PLpgSQL_package* pkg = NULL; MemoryContext oldContext = CurrentMemoryContext; @@ -1044,8 +1043,7 @@ Datum plpgsql_inline_handler(PG_FUNCTION_ARGS) FmgrInfo flinfo; Datum retval; int rc; - // PGSTAT_INIT_PLSQL_TIME_RECORD - int64 startTime = 0; + PGSTAT_INIT_TIME_RECORD(); bool needRecord = false; _PG_init(); diff --git a/src/gausskernel/cbb/instruments/statement/instr_statement.cpp b/src/gausskernel/cbb/instruments/statement/instr_statement.cpp index 9573f4b84..636e3e634 100755 --- a/src/gausskernel/cbb/instruments/statement/instr_statement.cpp +++ b/src/gausskernel/cbb/instruments/statement/instr_statement.cpp @@ -75,7 +75,7 @@ #define STATEMENT_DETAILS_HEAD_SIZE (1) /* [VERSION] */ #define INSTR_STMT_UNIX_DOMAIN_PORT (-1) -#define INSTR_STATEMENT_ATTRNUM 53 +#define INSTR_STATEMENT_ATTRNUM (44 + TOTAL_TIME_INFO_TYPES) /* support different areas in stmt detail column */ #define STATEMENT_DETAIL_TYPE_LEN (1) @@ -503,9 +503,10 @@ static HeapTuple GetStatementTuple(Relation rel, StatementStatContext* statement set_stmt_row_activity_cache_io(statementInfo, values, &i); /* time info */ - for (int num = 0; num < TOTAL_TIME_INFO_TYPES; num++) { - if (num == NET_SEND_TIME) + for (int num = 0; num < TOTAL_TIME_INFO_TYPES_P1; num++) { + if (num == NET_SEND_TIME) { continue; + } values[i++] = Int64GetDatum(statementInfo->timeModel[num]); } @@ -550,6 +551,11 @@ static HeapTuple GetStatementTuple(Relation rel, StatementStatContext* statement SET_TEXT_VALUES(statementInfo->trace_id, i++); set_stmt_advise(statementInfo, values, nulls, &i); + /* time info addition */ + values[i++] = Int64GetDatum(statementInfo->timeModel[NET_SEND_TIME]); + for (int num = TOTAL_TIME_INFO_TYPES_P1; num < TOTAL_TIME_INFO_TYPES; num++) { + values[i++] = Int64GetDatum(statementInfo->timeModel[num]); + } Assert(INSTR_STATEMENT_ATTRNUM == i); return heap_form_tuple(RelationGetDescr(rel), values, nulls); } diff --git a/src/gausskernel/cbb/instruments/unique_sql/instr_unique_sql.cpp b/src/gausskernel/cbb/instruments/unique_sql/instr_unique_sql.cpp index 7cae9d41f..4050899c7 100755 --- a/src/gausskernel/cbb/instruments/unique_sql/instr_unique_sql.cpp +++ b/src/gausskernel/cbb/instruments/unique_sql/instr_unique_sql.cpp @@ -1436,9 +1436,10 @@ static void create_tuple_entry(TupleDesc tupdesc) TupleDescInitEntry(tupdesc, (AttrNumber)++i, "n_soft_parse", INT8OID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber)++i, "n_hard_parse", INT8OID, -1, 0); - for (num = 0; num < TOTAL_TIME_INFO_TYPES; num++) { - if (num == NET_SEND_TIME) + for (num = 0; num < TOTAL_TIME_INFO_TYPES_P1; num++) { + if (num == NET_SEND_TIME) { continue; + } TupleDescInitEntry(tupdesc, (AttrNumber)++i, TimeInfoTypeName[num], INT8OID, -1, 0); } TupleDescInitEntry(tupdesc, (AttrNumber)++i, "NET_SEND_INFO", TEXTOID, -1, 0); @@ -1458,6 +1459,10 @@ static void create_tuple_entry(TupleDesc tupdesc) TupleDescInitEntry(tupdesc, (AttrNumber)++i, "hash_mem_used", INT8OID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber)++i, "hash_spill_count", INT8OID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber)++i, "hash_spill_size", INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)++i, TimeInfoTypeName[NET_SEND_TIME], INT8OID, -1, 0); + for (num = TOTAL_TIME_INFO_TYPES_P1; num < TOTAL_TIME_INFO_TYPES; num++) { + TupleDescInitEntry(tupdesc, (AttrNumber)++i, TimeInfoTypeName[num], INT8OID, -1, 0); + } } static void set_tuple_cn_node_name(UniqueSQL* unique_sql, Datum* values, int* i) @@ -1544,10 +1549,11 @@ static void set_tuple_value(UniqueSQL* unique_sql, Datum* values, bool* nulls, i values[i++] = Int64GetDatum(unique_sql->parse.soft_parse); values[i++] = Int64GetDatum(unique_sql->parse.hard_parse); - // time Info - for (num = 0; num < TOTAL_TIME_INFO_TYPES; num++) { - if (num == NET_SEND_TIME) + // time Info p1 + for (num = 0; num < TOTAL_TIME_INFO_TYPES_P1; num++) { + if (num == NET_SEND_TIME) { continue; + } values[i++] = Int64GetDatum(unique_sql->timeInfo.TimeInfoArray[num]); } int idx = 0; @@ -1576,6 +1582,11 @@ static void set_tuple_value(UniqueSQL* unique_sql, Datum* values, bool* nulls, i values[i++] = Int64GetDatum(unique_sql->hash_state.used_work_mem); values[i++] = Int64GetDatum(unique_sql->hash_state.spill_counts); values[i++] = Int64GetDatum(unique_sql->hash_state.spill_size); + // time Info + values[i++] = Int64GetDatum(unique_sql->timeInfo.TimeInfoArray[NET_SEND_TIME]); + for (num = TOTAL_TIME_INFO_TYPES_P1; num < TOTAL_TIME_INFO_TYPES; num++) { + values[i++] = Int64GetDatum(unique_sql->timeInfo.TimeInfoArray[num]); + } Assert(arr_size == i); } @@ -1614,7 +1625,7 @@ Datum get_instr_unique_sql(PG_FUNCTION_ARGS) { FuncCallContext* funcctx = NULL; long num = 0; -#define INSTRUMENTS_UNIQUE_SQL_ATTRNUM (35 + TOTAL_TIME_INFO_TYPES - 1) +#define INSTRUMENTS_UNIQUE_SQL_ATTRNUM (35 + TOTAL_TIME_INFO_TYPES) CheckVersion(); check_unique_sql_permission(); diff --git a/src/gausskernel/process/postmaster/Makefile b/src/gausskernel/process/postmaster/Makefile index 1fdb0c261..fc85d6d2d 100644 --- a/src/gausskernel/process/postmaster/Makefile +++ b/src/gausskernel/process/postmaster/Makefile @@ -31,7 +31,7 @@ ifneq "$(MAKECMDGOALS)" "clean" endif endif endif -OBJS = autovacuum.o bgwriter.o fork_process.o pgarch.o pgstat.o postmaster.o gaussdb_version.o\ +OBJS = autovacuum.o bgwriter.o fork_process.o pgarch.o pgstat.o og_record_time.o postmaster.o gaussdb_version.o\ startup.o syslogger.o walwriter.o walwriterauxiliary.o checkpointer.o pgaudit.o alarmchecker.o \ twophasecleaner.o fencedudf.o lwlockmonitor.o cbmwriter.o pagewriter.o pagerepair.o snapcapturer.o rbcleaner.o globalstats.o \ cfs_shrinker.o \ diff --git a/src/gausskernel/process/postmaster/og_record_time.cpp b/src/gausskernel/process/postmaster/og_record_time.cpp new file mode 100644 index 000000000..b27b0d42b --- /dev/null +++ b/src/gausskernel/process/postmaster/og_record_time.cpp @@ -0,0 +1,698 @@ +/* + * Copyright (c) 2023 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * --------------------------------------------------------------------------------------- + * + * IDENTIFICATION + * src/gausskernel/process/postmaster/og_record_time.cpp + * + * NOTES + * Implements some of this code used to record sql execute time. + * + * ------------------------------------------------------------------------- + */ + +#include "og_record_time.h" +#include "tcop/tcopprot.h" +const char* TimeInfoTypeName[] = { + "DB_TIME", + "CPU_TIME", + "EXECUTION_TIME", + "PARSE_TIME", + "PLAN_TIME", + "REWRITE_TIME", /*total elapsed time of rewrite stage.*/ + "PL_EXECUTION_TIME", /*total elapsed time of plpgsql exection.*/ + "PL_COMPILATION_TIME", /*total elapsed time of plpgsql compilation.*/ + "NET_SEND_TIME", + "DATA_IO_TIME", + "SRT1_Q", + "SRT2_SIMPLE_QUERY", + "SRT3_ANALYZE_REWRITE", + "SRT4_PLAN_QUERY", + "SRT5_LIGHT_QUERY", + "SRT6_P", + "SRT7_B", + "SRT8_E", + "SRT9_D", + "SRT10_S", + "SRT11_C", + "SRT12_U", + "SRT13_BEFORE_QUERY", + "SRT14_AFTER_QUERY", + "RTT_UNKNOWN", + "NET_SEND_TIMES", + "NET_SEND_N_CALLS", + "NET_SEND_SIZE", + "NET_RECV_TIMES", + "NET_RECV_N_CALLS", + "NET_RECV_SIZE", + "NET_STREAM_SEND_times", + "NET_STREAM_SEND_n_calls", + "NET_STREAM_SEND_size", + "NET_STREAM_RECV_times", + "NET_STREAM_RECV_n_calls", + "NET_STREAM_RECV_size" +}; + +static inline knl_u_stat_context* get_record_cxt() +{ + return &(u_sess->stat_cxt); +} + +static int og_get_time_record_level() +{ + Assert(u_sess != NULL); + return u_sess->attr.attr_common.time_record_level; +} + +OgRecordStat* og_get_record_stat() +{ + OgRecordStat* record_stat = (OgRecordStat*) get_record_cxt()->og_record_stat; + Assert(record_stat != NULL); + return record_stat; +} + + +void og_record_time_cleanup(int code, Datum arg) +{ + if (u_sess == NULL) { + return; + } + ereport(DEBUG1, + (errmsg("record(%ld): cleanup called!", + get_record_cxt()->og_record_stat == NULL ? -1 : ((int64)(get_record_cxt()->og_record_stat))))); + if (get_record_cxt()->og_record_stat != NULL) { + DELETE_EX_TYPE(get_record_cxt()->og_record_stat, OgRecordStat); + } +} + +void og_record_time_reinit() +{ + if (u_sess == NULL || get_record_cxt()->og_record_stat == NULL) { + return; + } + og_get_record_stat()->reinit(); + ResetMemory(u_sess->stat_cxt.localTimeInfoArray, + sizeof(int64) * TOTAL_TIME_INFO_TYPES); + ResetMemory(u_sess->stat_cxt.localNetInfo, + sizeof(uint64) * TOTAL_NET_INFO_TYPES); +} + +const char* og_record_time_type_str(const RecordType& time_type) +{ + return og_record_time_type_str(time_type.position()); +} + +const char* og_record_time_type_str(int pos) +{ + int max_size = sizeof(TimeInfoTypeName) / sizeof(TimeInfoTypeName[0]); + if (pos < 0 || pos >= max_size) { + pos = max_size - 1; + } + return TimeInfoTypeName[pos]; +} + +bool og_time_record_start() +{ + return og_get_record_stat()->start_first_record_opt(); +} + +bool og_time_record_end() +{ + return og_get_record_stat()->free_first_record_opt(); +} + +bool og_time_record_is_started() +{ + return og_get_record_stat()->already_start(); +} + +int64 og_get_time_unique_id() +{ + if (u_sess == NULL + || ((OgRecordStat*) get_record_cxt()->og_record_stat == NULL)) { + return 0; + } + return og_get_record_stat()->get_time_unique_id(); +} + +static inline void og_record_report_start(const OgTimeDataVo& record) +{ + og_get_record_stat()->report_start(record); +} + +static inline void og_record_report_end(const OgTimeDataVo& record) +{ + og_get_record_stat()->report_end(record); +} + +static inline void og_record_report_duplicate(OgTimeDataVo& record) +{ + og_get_record_stat()->report_duplicate(record); +} + +RecordType::RecordType() +{ + type_code = (int) RTT_UNKNOWN; + rtt_type = TIME_INFO; +} + +RecordType::RecordType(TimeInfoType time_info_type) +{ + type_code = (int) time_info_type; + rtt_type = TIME_INFO; +} +RecordType::RecordType(SelfRecordType self_typ) +{ + type_code = (int) self_typ; + rtt_type = SELF_INFO; +} + +RecordType::RecordType(NetInfoType net_info_type, ssize_t str_len) +{ + type_code = (int) net_info_type; + rtt_type = NET_INFO; + this->str_len = str_len; +} + +RecordTimeType RecordType::get_record_time_type() const +{ + return rtt_type; +} + +int RecordType::get_type_code() const +{ + return type_code; +} + +ssize_t RecordType::get_str_len() const +{ + return str_len; +} + +int RecordType::get_init_pos() const +{ + if (rtt_type == NET_INFO) { + return (int) TOTAL_TIME_INFO_TYPES + (int) SRT_ALL; + } else if (rtt_type == SELF_INFO) { + return (int) TOTAL_TIME_INFO_TYPES; + } + return 0; +} + +int RecordType::position() const +{ + return get_type_code() + get_init_pos(); +} + +bool RecordType::is_root_type() const +{ + return *this == DB_TIME; +} + +bool RecordType::operator==(TimeInfoType time_type) const +{ + if (rtt_type != TIME_INFO) { + return false; + } + return type_code == (int)time_type; +} +bool RecordType::operator!=(TimeInfoType time_type) const +{ + return !(*this == time_type); +} + +bool RecordType::operator==(NetInfoType net_type) const +{ + if (rtt_type != NET_INFO) { + return false; + } + return type_code == (int) net_type; +} + +bool RecordType::operator!=(NetInfoType net_type) const +{ + return !(*this == net_type); +} + +bool RecordType::operator==(SelfRecordType self_type) const +{ + if (rtt_type != SELF_INFO) { + return false; + } + return type_code == (int) self_type; +} + +bool RecordType::operator!=(SelfRecordType self_type) const +{ + return !(*this == self_type); +} + +const char* OgTimeDataFormatHelper::format(const OgTimeDataVo& vo) +{ + int ret = snprintf_s(format_str, DEFAULT_FORMAT_LENGTH, DEFAULT_FORMAT_LENGTH - 1, + "rd:{id=%lld,s=%lld,e=%lld,t=%lld,d=%d,na=%s}", vo.id, vo.begin, + vo.end == 0 ? GetCurrentTimestamp(): vo.end, vo.end == 0 ? 0 : vo.total(), + vo.depth, og_record_time_type_str(vo.record_type)); + securec_check_ss(ret, "\0", "\0"); + return format_str; +} + +OgRecordOperator::OgRecordOperator(bool auto_record) +{ + init(auto_record, RecordType(RTT_UNKNOWN)); +} + +OgRecordOperator::OgRecordOperator(TimeInfoType time_info_type) +{ + init(true, RecordType(time_info_type)); +} + +OgRecordOperator::OgRecordOperator(NetInfoType net_info_type) +{ + init(true, RecordType(net_info_type, 0)); +} + +OgRecordOperator::OgRecordOperator(SelfRecordType self_type) +{ + init(true, RecordType(self_type)); +} + +OgRecordOperator::OgRecordOperator(bool auto_record, NetInfoType net_type) { + init(auto_record, RecordType(net_type, 0)); +} + +OgRecordOperator::OgRecordOperator(bool auto_record, TimeInfoType time_type) { + init(auto_record, RecordType(time_type)); +} + +OgRecordOperator::OgRecordOperator(bool auto_record, SelfRecordType self_type) { + init(auto_record, RecordType(self_type)); +} + +OgRecordOperator::~OgRecordOperator() +{ + if (this->auto_record) { + exit(); + } +} + +void OgRecordOperator::enter(TimeInfoType time_type) +{ + enter(RecordType(time_type)); +} + +void OgRecordOperator::enter(NetInfoType net_type) +{ + enter(RecordType(net_type, 0)); +} + +void OgRecordOperator::enter(const RecordType& record_type) +{ + if (!report_enable()) { + return; + } + if (record_type != RTT_UNKNOWN) { + base_record.record_type = record_type; + } + if (!base_record.record_type.is_root_type()) { + base_record.begin = (int64)GetCurrentTimestamp(); + } + og_record_report_start(base_record); +} + +void OgRecordOperator::exit(TimeInfoType time_info_type) +{ + exit(RecordType(time_info_type)); +} + +void OgRecordOperator::exit(NetInfoType net_type, ssize_t str_len) +{ + exit(RecordType(net_type, str_len)); +} + +void OgRecordOperator::exit(const RecordType& record_type) +{ + if (!report_enable()) { + return; + } + if (record_type != RTT_UNKNOWN) { + base_record.record_type = record_type; + } + og_record_report_end(base_record); +} + +// this only called after exit() to report new record again. +void OgRecordOperator::report_duplicate(NetInfoType net_type, ssize_t str_len) +{ + RecordType old = base_record.record_type; + base_record.record_type = RecordType(net_type, str_len); + og_record_report_duplicate(base_record); + base_record.record_type = old; +} + +int64 OgRecordOperator::get_record_id() const +{ + return base_record.id; +} + +void OgRecordOperator::update_record_id() +{ + base_record.id = og_get_time_unique_id(); +} + +bool OgRecordOperator::report_enable() const +{ + OgRecordStat* record_stat = (OgRecordStat*)get_record_cxt()->og_record_stat; + if (record_stat == NULL) { + return false; + } + // for future, we can control diff level record time. + int level = record_stat->get_time_record_level(); + return level == 0; +} + +void OgRecordOperator::init(bool auto_record, const RecordType& record_type) +{ + this->auto_record = auto_record; + if (this->auto_record) { + enter(record_type); + } else { + base_record.record_type = record_type; + } +} + +OgTimeDataVo& OgTimeDataStack::top() +{ + Assert(cur_pos >= 0 && cur_pos < DEFAULT_TIME_DATA_STACK_DEPTH); + return data_list[cur_pos]; +} + +const OgTimeDataVo& OgTimeDataStack::top() const +{ + Assert(cur_pos >= 0 && cur_pos < DEFAULT_TIME_DATA_STACK_DEPTH); + return data_list[cur_pos]; +} + +bool OgTimeDataStack::empty() const +{ + return cur_pos < 0; +} + +size_t OgTimeDataStack::size() const { + Assert(cur_pos >= -1 && cur_pos < DEFAULT_TIME_DATA_STACK_DEPTH); + return cur_pos + 1; +} + +bool OgTimeDataStack::push(const OgTimeDataVo &vo) +{ + if (cur_pos == DEFAULT_TIME_DATA_STACK_DEPTH - 1) { + return false; + } + data_list[++ cur_pos] = vo; + return true; +} + +void OgTimeDataStack::pop() +{ + if (cur_pos < 0) { + return; + } + cur_pos --; +} + +void OgTimeDataStack::reset() +{ + cur_pos = -1; +} + +OgRecordStat::OgRecordStat(int64* local_time_info, uint64* loca_net_info) +:first_record_opt(false, DB_TIME) +{ + log_trace_msg = makeStringInfo(); + record_start = false; + this->local_time_info = local_time_info; + this->local_net_info = loca_net_info; + time_unique_id = 0; + db_time_baseline = DEFAULT_DB_TIME_BASELINE; + time_record_level = 0; + reset(); +} + +OgRecordStat::~OgRecordStat() +{ + Destroy(); +} + +void OgRecordStat::Destroy() +{ + reset(); + if (log_trace_msg != NULL) { + DestroyStringInfo(log_trace_msg); + log_trace_msg = NULL; + } +} + +void OgRecordStat::reset() +{ + this->logtrace(DEBUG1, "reset"); + depth = INVALID_DEPTH; + record_start = false; + free_first_record_opt(); + records_stack.reset(); + records_stack.push(OgTimeDataVo()); +} + +void OgRecordStat::reinit() +{ + this->logtrace(DEBUG1, "reinit"); + reset(); + pre_records_stack.reset(); +} + +void OgRecordStat::report_start(const OgTimeDataVo& data_record) +{ + if (!records_stack.push(data_record)) { + return; + } + records_stack.top().depth = increment_depth(); + record_debug("begin: %s", FORMAT_VO(records_stack.top())); + if (data_record.record_type.is_root_type()) { + records_stack.top().begin = GetCurrentTimestamp(); + } +} + +void OgRecordStat::report_end(const OgTimeDataVo& record) +{ + // assert not records_stack.is_empty() + OgTimeDataVo& time_vo = records_stack.top(); + if (record != time_vo) { + record_debug("find time record not match!top =%s, curr=%s", + FORMAT_VO(time_vo), FORMAT_VO(record)); + return; + } + records_stack.pop(); + decrement_depth(); + if (!already_start()) { + return; + } + time_vo.record_type = record.record_type; + if (time_vo.record_type.is_root_type()) { + // We don't want debug time calc in root type. + time_vo.end = GetCurrentTimestamp(); + record_debug("end : %s", FORMAT_VO(time_vo)); + } else { + record_debug("end : %s", FORMAT_VO(time_vo)); + time_vo.end = GetCurrentTimestamp(); + } + OgTimeDataVo& parent_time_vo = records_stack.top(); + int64 cost = time_vo.cost(); + int64 total = time_vo.total(); + if (time_vo.record_type.get_record_time_type() != NET_INFO) { + parent_time_vo.update_other_cost(total); + } + update_record_time(time_vo.record_type, cost); +} + +// some auto report can only report net record. +void OgRecordStat::report_duplicate(OgTimeDataVo& record) +{ + Assert(record.record_type.get_record_time_type() == NET_INFO); + record_debug("duplicate : %s, str_len=%d", FORMAT_VO(record), record.record_type.get_str_len()); + if (record.end == 0) { + record.end = GetCurrentTimestamp(); + } + update_record_time(record.record_type, record.cost()); +} + +int OgRecordStat::increment_depth() +{ + return ++ depth; +} + +void OgRecordStat::decrement_depth() +{ + depth --; +} + +int64 OgRecordStat::get_time_unique_id() +{ + time_unique_id ++; + return time_unique_id; +} + +void OgRecordStat::print_self() const +{ + int64 bind_total = 0; + int64 bind_data = 0; + if (!log_enable_debug()) { + return; + } + for (int i = 0; i < TOTAL_RECORD_TYPES; i ++) + { + if (i < TOTAL_TIME_INFO_TYPES) { + bind_total += local_time_info[i]; + bind_data = local_time_info[i]; + } else if(i < TOTAL_TIME_INFO_TYPES + SRT_ALL) { + bind_data = 0; + } else if(i < TOTAL_RECORD_TYPES) { + int tmp_pos = i - TOTAL_TIME_INFO_TYPES - SRT_ALL; + bind_data = local_net_info[tmp_pos]; + } else { + // nothing to do + } + record_debug("name:%d %30s %ld", i, og_record_time_type_str(i), bind_data); + } + int64 child_cost = bind_total - get_record_times(DB_TIME) - get_record_times(CPU_TIME); + record_debug("%s,%s,diff(%lld) = total=(%lld) - child_cost(%lld)", + get_record_times(RTT_UNKNOWN) < db_time_baseline ? "rd_yes" : "rd_no ", + (child_cost == get_db_time()) ? "rd_eq " : "rd_ne", + get_record_times(RTT_UNKNOWN), + get_db_time(), child_cost); +} + +int OgRecordStat::get_time_record_level() const +{ + return time_record_level; +} + +void OgRecordStat::update_time_record_level() +{ + time_record_level = og_get_time_record_level(); +} + +bool OgRecordStat::start_first_record_opt() +{ + if (!already_start()) { + reset(); + update_time_record_level(); + first_record_opt.update_record_id(); + first_record_opt.enter(DB_TIME); + record_start = true; + while (!pre_records_stack.empty()) { + OgTimeDataVo& vo = pre_records_stack.top(); + pre_records_stack.pop(); + vo.begin = GetCurrentTimestamp(); + vo.other_cost = 0; + report_start(vo); + } + return true; + } + return false; +} + +bool OgRecordStat::free_first_record_opt() +{ + if(already_start()) { + int64 record_id = first_record_opt.get_record_id(); + while (records_stack.size() > 2 && (records_stack.top().id != record_id)) { + records_stack.top().end = GetCurrentTimestamp(); + pre_records_stack.push(records_stack.top()); + logtrace(DEBUG1, "free_head, %s", OgTimeDataFormatHelper().format(records_stack.top())); + report_end(records_stack.top()); + } + first_record_opt.exit(); + record_start = false; + logtrace(DEBUG1, "free_first_record_opt called, %d", record_id); + local_time_info[RTT_UNKNOWN] += local_time_info[DB_TIME]; + local_time_info[DB_TIME] = get_db_time(); + return true; + } + return false; +} + +const int64 OgRecordStat::get_record_times(TimeInfoType type_info) const +{ + return local_time_info[type_info]; +} + +const int64 OgRecordStat::get_db_time() const +{ + return records_stack.top().other_cost; +} + +const OgTimeDataVo& OgRecordStat::get_root_time_data_vo() const +{ + return records_stack.top(); +} + +inline bool OgRecordStat::already_start() const +{ + return record_start; +} + +void OgRecordStat::update_record_time(const RecordType& record_type, int64 cost) +{ + if (record_type.get_record_time_type() == TIME_INFO) { + local_time_info[record_type.position()] += cost; + } else if (record_type.get_record_time_type() == SELF_INFO) { + // not use, only for add new time record to quick debug + } else { + ssize_t str_len = record_type.get_str_len(); + if (str_len != 0) { + int type_code = record_type.get_type_code(); + local_net_info[type_code] += (uint64)cost; + local_net_info[type_code + 1] ++; + local_net_info[type_code + 2] += str_len; + } + } +} + +// flow database system log config +bool OgRecordStat::log_enable() const +{ + return u_sess != NULL && u_sess->attr.attr_common.log_statement == LOGSTMT_ALL; +} + +// flow database system log config +bool OgRecordStat::log_enable_debug() const +{ + return log_enable() && u_sess->attr.attr_common.log_min_messages < LOG; +} + +void OgRecordStat::logtrace(int level, const char* fmt, ...) const +{ + if (!log_enable() || log_trace_msg == NULL) { + return; + } + if (fmt != log_trace_msg->data) { + va_list args; + (void)va_start(args, fmt); + // This place just is the message print. So there is't need check the value of vsnprintf_s function return. if + // checked, when the message lengtn is over than log_trace_msg->maxlen, will be abnormal exit. + (void)vsnprintf_s(log_trace_msg->data, log_trace_msg->maxlen, log_trace_msg->maxlen - 1, fmt, args); + va_end(args); + } + ereport(level, (errmsg("record(%ld-%ld) %s", (uint64)this, + u_sess == NULL ? 0 : u_sess->session_id, log_trace_msg->data))); +} diff --git a/src/gausskernel/process/postmaster/pgstat.cpp b/src/gausskernel/process/postmaster/pgstat.cpp index 6b0b2f54a..173449517 100644 --- a/src/gausskernel/process/postmaster/pgstat.cpp +++ b/src/gausskernel/process/postmaster/pgstat.cpp @@ -8162,17 +8162,6 @@ Size sessionTimeShmemSize(void) return mul_size(SessionTimeArraySize, sizeof(SessionTimeEntry)); } -const char* TimeInfoTypeName[TOTAL_TIME_INFO_TYPES] = {"DB_TIME", - "CPU_TIME", - "EXECUTION_TIME", - "PARSE_TIME", - "PLAN_TIME", - "REWRITE_TIME", - "PL_EXECUTION_TIME", - "PL_COMPILATION_TIME", - "NET_SEND_TIME", - "DATA_IO_TIME"}; - void sessionTimeShmemInit(void) { bool found = false; @@ -8307,68 +8296,60 @@ void AttachMySessionTimeEntry(void) Assert((t_thrd.shemem_ptr_cxt.mySessionTimeEntry->changeCount & 1) == 0); } -void timeInfoRecordStart(void) +static void addThreadTimeEntry() { - if (u_sess->stat_cxt.localTimeInfoArray[DB_TIME] == 0) { - u_sess->stat_cxt.localTimeInfoArray[DB_TIME] = GetCurrentTimestamp(); - if (u_sess->attr.attr_common.enable_instr_cpu_timer) - u_sess->stat_cxt.localTimeInfoArray[CPU_TIME] = getCpuTime(); + for (int i = 0; i < TOTAL_TIME_INFO_TYPES; i++) { + t_thrd.shemem_ptr_cxt.mySessionTimeEntry->array[i] += + u_sess->stat_cxt.localTimeInfoArray[i]; } } - -void timeInfoRecordEnd(void) +void ResetMemory(void* dest, size_t size) { errno_t rc; - if (u_sess->stat_cxt.localTimeInfoArray[DB_TIME] != 0) { - t_thrd.shemem_ptr_cxt.mySessionTimeEntry->changeCount++; - - if (u_sess->attr.attr_common.enable_instr_cpu_timer) { - int64 cur = getCpuTime(); + rc = memset_s(dest, size, 0, size); + securec_check(rc, "\0", "\0"); +} - u_sess->stat_cxt.localTimeInfoArray[CPU_TIME] = cur - u_sess->stat_cxt.localTimeInfoArray[CPU_TIME]; - } - u_sess->stat_cxt.localTimeInfoArray[DB_TIME] = - GetCurrentTimestamp() - u_sess->stat_cxt.localTimeInfoArray[DB_TIME]; +void timeInfoRecordStart(void) +{ + if (!og_time_record_start()) { + return; + } + if (u_sess->attr.attr_common.enable_instr_cpu_timer) + u_sess->stat_cxt.localTimeInfoArray[CPU_TIME] = getCpuTime(); +} - t_thrd.shemem_ptr_cxt.mySessionTimeEntry->array[CPU_TIME] += u_sess->stat_cxt.localTimeInfoArray[CPU_TIME]; - t_thrd.shemem_ptr_cxt.mySessionTimeEntry->array[DB_TIME] += u_sess->stat_cxt.localTimeInfoArray[DB_TIME]; +void timeInfoRecordEnd(void) +{ + if (!og_time_record_is_started()) { + return; + } + t_thrd.shemem_ptr_cxt.mySessionTimeEntry->changeCount++; - t_thrd.shemem_ptr_cxt.mySessionTimeEntry->array[EXECUTION_TIME] += - u_sess->stat_cxt.localTimeInfoArray[EXECUTION_TIME]; - t_thrd.shemem_ptr_cxt.mySessionTimeEntry->array[PARSE_TIME] += u_sess->stat_cxt.localTimeInfoArray[PARSE_TIME]; - t_thrd.shemem_ptr_cxt.mySessionTimeEntry->array[PLAN_TIME] += u_sess->stat_cxt.localTimeInfoArray[PLAN_TIME]; - t_thrd.shemem_ptr_cxt.mySessionTimeEntry->array[REWRITE_TIME] += - u_sess->stat_cxt.localTimeInfoArray[REWRITE_TIME]; + if (u_sess->attr.attr_common.enable_instr_cpu_timer) { + int64 cur = getCpuTime(); + u_sess->stat_cxt.localTimeInfoArray[CPU_TIME] = cur - + u_sess->stat_cxt.localTimeInfoArray[CPU_TIME]; + } + og_time_record_end(); + og_get_record_stat()->print_self(); - t_thrd.shemem_ptr_cxt.mySessionTimeEntry->array[PL_EXECUTION_TIME] += - u_sess->stat_cxt.localTimeInfoArray[PL_EXECUTION_TIME]; - t_thrd.shemem_ptr_cxt.mySessionTimeEntry->array[PL_COMPILATION_TIME] += - u_sess->stat_cxt.localTimeInfoArray[PL_COMPILATION_TIME]; + addThreadTimeEntry(); + t_thrd.shemem_ptr_cxt.mySessionTimeEntry->changeCount++; + Assert((t_thrd.shemem_ptr_cxt.mySessionTimeEntry->changeCount & 1) == 0); - t_thrd.shemem_ptr_cxt.mySessionTimeEntry->array[NET_SEND_TIME] += - u_sess->stat_cxt.localTimeInfoArray[NET_SEND_TIME]; - t_thrd.shemem_ptr_cxt.mySessionTimeEntry->array[DATA_IO_TIME] += - u_sess->stat_cxt.localTimeInfoArray[DATA_IO_TIME]; + UniqueSQLStat sqlStat; + sqlStat.timeInfo = u_sess->stat_cxt.localTimeInfoArray; + sqlStat.netInfo = u_sess->stat_cxt.localNetInfo; + if (u_sess->unique_sql_cxt.unique_sql_id != 0 && is_unique_sql_enabled()) { + UpdateUniqueSQLStat(NULL, NULL, 0, NULL, &sqlStat); - t_thrd.shemem_ptr_cxt.mySessionTimeEntry->changeCount++; - Assert((t_thrd.shemem_ptr_cxt.mySessionTimeEntry->changeCount & 1) == 0); - - UniqueSQLStat sqlStat; - sqlStat.timeInfo = u_sess->stat_cxt.localTimeInfoArray; - sqlStat.netInfo = u_sess->stat_cxt.localNetInfo; - if (u_sess->unique_sql_cxt.unique_sql_id != 0 && is_unique_sql_enabled()) - UpdateUniqueSQLStat(NULL, NULL, 0, NULL, &sqlStat); - rc = memset_s(u_sess->stat_cxt.localTimeInfoArray, - sizeof(int64) * TOTAL_TIME_INFO_TYPES, - 0, - sizeof(int64) * TOTAL_TIME_INFO_TYPES); - securec_check(rc, "\0", "\0"); - rc = memset_s(u_sess->stat_cxt.localNetInfo, - sizeof(uint64) * TOTAL_NET_INFO_TYPES, - 0, - sizeof(uint64) * TOTAL_NET_INFO_TYPES); - securec_check(rc, "\0", "\0"); } + + ResetMemory(u_sess->stat_cxt.localTimeInfoArray, + sizeof(int64) * TOTAL_TIME_INFO_TYPES); + ResetMemory(u_sess->stat_cxt.localNetInfo, + sizeof(uint64) * TOTAL_NET_INFO_TYPES); } /* generate result of pv_session_time view */ diff --git a/src/gausskernel/process/tcop/postgres.cpp b/src/gausskernel/process/tcop/postgres.cpp index 3954e68ab..6fae4e501 100755 --- a/src/gausskernel/process/tcop/postgres.cpp +++ b/src/gausskernel/process/tcop/postgres.cpp @@ -1023,15 +1023,13 @@ List* pg_parse_query(const char* query_string, List** query_string_locationlist, List* (*parser_hook)(const char*, List**)) { List* raw_parsetree_list = NULL; - PGSTAT_INIT_TIME_RECORD(); + OgRecordOperator _local_opt(PARSE_TIME); TRACE_POSTGRESQL_QUERY_PARSE_START(query_string); if (u_sess->attr.attr_common.log_parser_stats) ResetUsage(); - PGSTAT_START_TIME_RECORD(); - if (parser_hook == NULL) { parser_hook = raw_parser; #if (!defined(ENABLE_MULTIPLE_NODES)) && (!defined(ENABLE_PRIVATEGAUSS)) @@ -1047,7 +1045,6 @@ List* pg_parse_query(const char* query_string, List** query_string_locationlist, if (u_sess->parser_cxt.hasPartitionComment) { ereport(WARNING, (errmsg("comment is not allowed in partition/subpartition."))); } - PGSTAT_END_TIME_RECORD(PARSE_TIME); if (u_sess->attr.attr_common.log_parser_stats) ShowUsage("PARSER STATISTICS"); @@ -1082,6 +1079,7 @@ List* pg_parse_query(const char* query_string, List** query_string_locationlist, */ List* pg_analyze_and_rewrite(Node* parsetree, const char* query_string, Oid* paramTypes, int numParams) { + OgRecordOperator _local_opt(SRT3_ANALYZE_REWRITE); Query* query = NULL; List* querytree_list = NULL; @@ -1362,7 +1360,7 @@ static void check_query_acl(Query* query) PlannedStmt* pg_plan_query(Query* querytree, int cursorOptions, ParamListInfo boundParams, bool underExplain) { PlannedStmt* plan = NULL; - PGSTAT_INIT_TIME_RECORD(); + OgRecordOperator _local_opt(PLAN_TIME); bool multi_node_hint = false; /* Utility commands have no plans. */ @@ -1385,8 +1383,6 @@ PlannedStmt* pg_plan_query(Query* querytree, int cursorOptions, ParamListInfo bo /* Update hard parse counter for Unique SQL */ UniqueSQLStatCountHardParse(1); - PGSTAT_START_TIME_RECORD(); - /* check perssion for expect_computing_nodegroup */ if (!OidIsValid(lc_replan_nodegroup)) check_query_acl(querytree); @@ -1398,8 +1394,6 @@ PlannedStmt* pg_plan_query(Query* querytree, int cursorOptions, ParamListInfo bo plan = planner(querytree, cursorOptions, boundParams); } - PGSTAT_END_TIME_RECORD(PLAN_TIME); - if (u_sess->attr.attr_common.log_planner_stats) ShowUsage("PLANNER STATISTICS"); @@ -1492,6 +1486,7 @@ __attribute__((unused)) static bool is_insert_multiple_values_query_in_gtmfree(Q */ List* pg_plan_queries(List* querytrees, int cursorOptions, ParamListInfo boundParams) { + OgRecordOperator _local_opt(SRT4_PLAN_QUERY); List* stmt_list = NIL; ListCell* query_list = NULL; @@ -2377,6 +2372,7 @@ bool IsRightRefState(List* plantreeList) */ static void exec_simple_query(const char* query_string, MessageType messageType, StringInfo msg = NULL) { + OgRecordOperator _local_opt(SRT2_SIMPLE_QUERY); CommandDest dest = (CommandDest)t_thrd.postgres_cxt.whereToSendOutput; MemoryContext oldcontext; MemoryContext OptimizerContext; @@ -8404,6 +8400,10 @@ int PostgresMain(int argc, char* argv[], const char* dbname, const char* usernam if (AlignMemoryContext != NULL) MemoryContextReset(AlignMemoryContext); + + // reinit record time class and related memory. + og_record_time_reinit(); + /* * Now return to normal top-level context and clear ErrorContext for * next time. @@ -8482,10 +8482,15 @@ int PostgresMain(int argc, char* argv[], const char* dbname, const char* usernam } bool template0_locked = false; + OgRecordOperator _local_tmp_opt(false, SRT13_BEFORE_QUERY); + OgRecordOperator _local_tmp_opt1(false, SRT14_AFTER_QUERY); /* * Non-error queries loop here. */ for (;;) { + if (og_time_record_is_started()) { + _local_tmp_opt.enter(); + } /* * Since max_query_rerty_times is a USERSET GUC, so must check Statement retry * in each query loop here. @@ -8626,6 +8631,9 @@ int PostgresMain(int argc, char* argv[], const char* dbname, const char* usernam } #endif /* update our elapsed time statistics. */ + if (og_time_record_is_started()) { + _local_tmp_opt.exit(); + } timeInfoRecordEnd(); /* reset unique_sql_id & stat @@ -8642,6 +8650,9 @@ int PostgresMain(int argc, char* argv[], const char* dbname, const char* usernam send_ready_for_query = false; } else { /* update our elapsed time statistics. */ + if (og_time_record_is_started()) { + _local_tmp_opt.exit(); + } timeInfoRecordEnd(); } /* @@ -8742,6 +8753,7 @@ int PostgresMain(int argc, char* argv[], const char* dbname, const char* usernam firstchar = ReadCommand(&input_message); /* update our elapsed time statistics. */ timeInfoRecordStart(); + _local_tmp_opt1.enter(); /* stmt retry routine phase : pack input_message */ if (IsStmtRetryEnabled()) { @@ -8786,6 +8798,7 @@ int PostgresMain(int argc, char* argv[], const char* dbname, const char* usernam u_sess->proc_cxt.MyProcPort->protocol_config->fn_process_command) { firstchar = u_sess->proc_cxt.MyProcPort->protocol_config->fn_process_command(&input_message); send_ready_for_query = true; + _local_tmp_opt1.exit(); continue; } @@ -8793,8 +8806,10 @@ int PostgresMain(int argc, char* argv[], const char* dbname, const char* usernam * (7) process the command. But ignore it if we're skipping till * Sync. */ - if (u_sess->postgres_cxt.ignore_till_sync && firstchar != EOF) + if (u_sess->postgres_cxt.ignore_till_sync && firstchar != EOF) { + _local_tmp_opt1.exit(); continue; + } #ifdef ENABLE_MULTIPLE_NODES // reset some flag related to stream ResetSessionEnv(); @@ -8837,8 +8852,10 @@ int PostgresMain(int argc, char* argv[], const char* dbname, const char* usernam firstchar); if (libpqsw_process_message(firstchar, &input_message)) { + _local_tmp_opt1.exit(); continue; } + _local_tmp_opt1.exit(); switch (firstchar) { #ifdef ENABLE_MULTIPLE_NODES @@ -8992,6 +9009,7 @@ int PostgresMain(int argc, char* argv[], const char* dbname, const char* usernam case 'Q': /* simple query */ { const char* query_string = NULL; + OgRecordOperator _local_opt(SRT1_Q); pgstat_report_trace_id(&u_sess->trace_cxt, true); query_string = pq_getmsgstring(&input_message); @@ -9361,6 +9379,7 @@ int PostgresMain(int argc, char* argv[], const char* dbname, const char* usernam #endif case 'P': /* parse */ { + OgRecordOperator _local_opt(SRT6_P); const char* stmt_name = NULL; const char* query_string = NULL; int numParams; @@ -9467,6 +9486,8 @@ int PostgresMain(int argc, char* argv[], const char* dbname, const char* usernam } break; case 'B': /* bind */ + { + OgRecordOperator _local_opt(SRT7_B); #ifdef USE_RETRY_STUB if (IsStmtRetryEnabled()) u_sess->exec_cxt.RetryController->stub_.StartOneStubTest(firstchar); @@ -9479,10 +9500,11 @@ int PostgresMain(int argc, char* argv[], const char* dbname, const char* usernam */ instr_stmt_report_trace_id(u_sess->trace_cxt.trace_id); exec_bind_message(&input_message); - break; + } break; case 'E': /* execute */ { + OgRecordOperator _local_opt(SRT8_E); const char* portal_name = NULL; int max_rows; @@ -9627,6 +9649,7 @@ int PostgresMain(int argc, char* argv[], const char* dbname, const char* usernam case 'C': /* close */ { + OgRecordOperator _local_opt(SRT11_C); int close_type; const char* closeTarget = NULL; @@ -9689,6 +9712,7 @@ int PostgresMain(int argc, char* argv[], const char* dbname, const char* usernam case 'D': /* describe */ { + OgRecordOperator _local_opt(SRT9_D); int describe_type; const char* describe_target = NULL; if ((unsigned int)input_message.len > SECUREC_MEM_MAX_LEN) { @@ -9741,6 +9765,8 @@ int PostgresMain(int argc, char* argv[], const char* dbname, const char* usernam break; case 'S': /* sync */ + { + OgRecordOperator _local_opt(SRT10_S); pq_getmsgend(&input_message); #ifdef USE_RETRY_STUB if (IsStmtRetryEnabled()) { @@ -9759,8 +9785,7 @@ int PostgresMain(int argc, char* argv[], const char* dbname, const char* usernam if (IsStmtRetryEnabled()) { t_thrd.log_cxt.flush_message_immediately = false; } - - break; + } break; /* * 'X' means that the frontend is closing down the socket. EOF @@ -10222,6 +10247,7 @@ int PostgresMain(int argc, char* argv[], const char* dbname, const char* usernam case 'U': /* msg type for batch Bind-Execute for PBE */ { + OgRecordOperator _local_opt(SRT12_U); if (!u_sess->attr.attr_common.support_batch_bind) ereport(ERROR, (errcode(ERRCODE_SYSTEM_ERROR), diff --git a/src/gausskernel/process/tcop/pquery.cpp b/src/gausskernel/process/tcop/pquery.cpp index 69c272945..d0bb9fcc0 100644 --- a/src/gausskernel/process/tcop/pquery.cpp +++ b/src/gausskernel/process/tcop/pquery.cpp @@ -1011,6 +1011,7 @@ bool PortalRun( { gstrace_entry(GS_TRC_ID_PortalRun); increase_instr_portal_nesting_level(); + OgRecordOperator _local_opt(EXECUTION_TIME); bool result = false; uint64 nprocessed; @@ -1759,6 +1760,7 @@ static void PortalRunMulti( { bool active_snapshot_set = false; ListCell* stmtlist_item = NULL; + OgRecordOperator _local_opt(EXECUTION_TIME); PGSTAT_INIT_TIME_RECORD(); #ifdef PGXC CombineTag combine; diff --git a/src/gausskernel/process/threadpool/knl_session.cpp b/src/gausskernel/process/threadpool/knl_session.cpp index 1752b26ab..55b942021 100755 --- a/src/gausskernel/process/threadpool/knl_session.cpp +++ b/src/gausskernel/process/threadpool/knl_session.cpp @@ -916,6 +916,8 @@ static void knl_u_stat_init(knl_u_stat_context* stat_cxt) size = sizeof(int64) * TOTAL_TIME_INFO_TYPES; stat_cxt->localTimeInfoArray = (int64*)palloc0(size); stat_cxt->localNetInfo = (uint64*)palloc0(sizeof(uint64) * TOTAL_NET_INFO_TYPES); + stat_cxt->og_record_stat = New(CurrentMemoryContext) OgRecordStat(stat_cxt->localTimeInfoArray, + stat_cxt->localNetInfo); stat_cxt->trackedMemChunks = 0; stat_cxt->trackedBytes = 0; diff --git a/src/gausskernel/runtime/executor/lightProxy.cpp b/src/gausskernel/runtime/executor/lightProxy.cpp index 1b279b7b4..1d34da09d 100644 --- a/src/gausskernel/runtime/executor/lightProxy.cpp +++ b/src/gausskernel/runtime/executor/lightProxy.cpp @@ -1336,6 +1336,7 @@ bool IsLightProxyOn(void) bool exec_query_through_light_proxy(List* querytree_list, Node* parsetree, bool snapshot_set, StringInfo msg, MemoryContext OptimizerContext) { + OgRecordOperator _local_opt(SRT5_LIGHT_QUERY); if ((list_length(querytree_list) == 1) && !IsA(parsetree, CreateTableAsStmt) && !IsA(parsetree, RefreshMatViewStmt)) { ExecNodes* single_exec_node = NULL; diff --git a/src/gausskernel/storage/ipc/ipc.cpp b/src/gausskernel/storage/ipc/ipc.cpp index 789899ce2..b35315af5 100644 --- a/src/gausskernel/storage/ipc/ipc.cpp +++ b/src/gausskernel/storage/ipc/ipc.cpp @@ -97,7 +97,8 @@ static const pg_on_exit_callback on_sess_exit_list[] = { AtProcExit_Files, audit_processlogout, log_disconnections, - libpqsw_cleanup + libpqsw_cleanup, + og_record_time_cleanup }; static const int on_sess_exit_size = lengthof(on_sess_exit_list); diff --git a/src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback-post_catalog_maindb_92_906.sql b/src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback-post_catalog_maindb_92_906.sql index d1f556cdc..c03aaa2e2 100644 --- a/src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback-post_catalog_maindb_92_906.sql +++ b/src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback-post_catalog_maindb_92_906.sql @@ -1 +1,956 @@ DROP FUNCTION IF EXISTS pg_catalog.ss_txnstatus_cache_stat() CASCADE; + +-- those type of view and producer will recreate. +-- openGauss=# select table_schema, table_name,column_name from information_schema.columns where column_name = 'db_time'; +-- table_schema | table_name | column_name +-- --------------+-------------------------------+------------- +-- pg_catalog | gs_wlm_session_query_info_all | db_time -- table +-- pg_catalog | gs_wlm_session_info_all | db_time -- view +-- dbe_perf | statement | db_time -- view +-- pg_catalog | statement_history | db_time -- unlogged table +-- dbe_perf | statement_history | db_time -- view +-- dbe_perf | summary_statement | db_time -- view +-- dbe_perf | gs_slow_query_info | db_time -- view +-- dbe_perf | gs_slow_query_history | db_time -- view +-- dbe_perf | global_slow_query_history | db_time -- view +-- dbe_perf | global_slow_query_info | db_time -- view +-- those proc will recreated. +-- openGauss=# select proname, pronamespace from pg_proc where proargnames @> array['db_time']; +-- proname | pronamespace +-- ----------------------------------+-------------- +-- get_instr_unique_sql | 11 +-- pg_stat_get_wlm_session_info | 11 +-- standby_statement_history | 4988 +-- standby_statement_history | 4988 +-- get_global_full_sql_by_timestamp | 4988 +-- get_global_slow_sql_by_timestamp | 4988 + +/* We process history related tables、views and function now. */ +DROP VIEW IF EXISTS DBE_PERF.statement CASCADE; +DROP VIEW IF EXISTS DBE_PERF.summary_statement CASCADE; +DROP VIEW IF EXISTS DBE_PERF.statement_history CASCADE; +DROP FUNCTION IF EXISTS pg_catalog.get_instr_unique_sql() CASCADE; +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 5702; +CREATE FUNCTION pg_catalog.get_instr_unique_sql +( + OUT node_name name, + OUT node_id integer, + OUT user_name name, + OUT user_id oid, + OUT unique_sql_id bigint, + OUT query text, + OUT n_calls bigint, + OUT min_elapse_time bigint, + OUT max_elapse_time bigint, + OUT total_elapse_time bigint, + OUT n_returned_rows bigint, + OUT n_tuples_fetched bigint, + OUT n_tuples_returned bigint, + OUT n_tuples_inserted bigint, + OUT n_tuples_updated bigint, + OUT n_tuples_deleted bigint, + OUT n_blocks_fetched bigint, + OUT n_blocks_hit bigint, + OUT n_soft_parse bigint, + OUT n_hard_parse bigint, + OUT db_time bigint, + OUT cpu_time bigint, + OUT execution_time bigint, + OUT parse_time bigint, + OUT plan_time bigint, + OUT rewrite_time bigint, + OUT pl_execution_time bigint, + OUT pl_compilation_time bigint, + OUT data_io_time bigint, + OUT net_send_info text, + Out net_recv_info text, + OUT net_stream_send_info text, + OUT net_stream_recv_info text, + OUT last_updated timestamp with time zone, + OUT sort_count bigint, + OUT sort_time bigint, + OUT sort_mem_used bigint, + OUT sort_spill_count bigint, + OUT sort_spill_size bigint, + OUT hash_count bigint, + OUT hash_time bigint, + OUT hash_mem_used bigint, + OUT hash_spill_count bigint, + OUT hash_spill_size bigint +) +RETURNS setof record LANGUAGE INTERNAL VOLATILE NOT FENCED as 'get_instr_unique_sql'; + +DROP INDEX IF EXISTS pg_catalog.statement_history_time_idx; +DROP TABLE IF EXISTS pg_catalog.statement_history cascade; + +CREATE unlogged table IF NOT EXISTS pg_catalog.statement_history( + db_name name, + schema_name name, + origin_node integer, + user_name name, + application_name text, + client_addr text, + client_port integer, + unique_query_id bigint, + debug_query_id bigint, + query text, + start_time timestamp with time zone, + finish_time timestamp with time zone, + slow_sql_threshold bigint, + transaction_id bigint, + thread_id bigint, + session_id bigint, + n_soft_parse bigint, + n_hard_parse bigint, + query_plan text, + n_returned_rows bigint, + n_tuples_fetched bigint, + n_tuples_returned bigint, + n_tuples_inserted bigint, + n_tuples_updated bigint, + n_tuples_deleted bigint, + n_blocks_fetched bigint, + n_blocks_hit bigint, + db_time bigint, + cpu_time bigint, + execution_time bigint, + parse_time bigint, + plan_time bigint, + rewrite_time bigint, + pl_execution_time bigint, + pl_compilation_time bigint, + data_io_time bigint, + net_send_info text, + net_recv_info text, + net_stream_send_info text, + net_stream_recv_info text, + lock_count bigint, + lock_time bigint, + lock_wait_count bigint, + lock_wait_time bigint, + lock_max_count bigint, + lwlock_count bigint, + lwlock_wait_count bigint, + lwlock_time bigint, + lwlock_wait_time bigint, + details bytea, + is_slow_sql boolean, + trace_id text, + advise text +); +REVOKE ALL on table pg_catalog.statement_history FROM public; +create index pg_catalog.statement_history_time_idx on pg_catalog.statement_history USING btree (start_time, is_slow_sql); + +DROP FUNCTION IF EXISTS dbe_perf.standby_statement_history(boolean); +DROP FUNCTION IF EXISTS dbe_perf.standby_statement_history(boolean, timestamp with time zone[]); +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 3118; +CREATE FUNCTION dbe_perf.standby_statement_history( + IN only_slow boolean, + OUT db_name name, + OUT schema_name name, + OUT origin_node integer, + OUT user_name name, + OUT application_name text, + OUT client_addr text, + OUT client_port integer, + OUT unique_query_id bigint, + OUT debug_query_id bigint, + OUT query text, + OUT start_time timestamp with time zone, + OUT finish_time timestamp with time zone, + OUT slow_sql_threshold bigint, + OUT transaction_id bigint, + OUT thread_id bigint, + OUT session_id bigint, + OUT n_soft_parse bigint, + OUT n_hard_parse bigint, + OUT query_plan text, + OUT n_returned_rows bigint, + OUT n_tuples_fetched bigint, + OUT n_tuples_returned bigint, + OUT n_tuples_inserted bigint, + OUT n_tuples_updated bigint, + OUT n_tuples_deleted bigint, + OUT n_blocks_fetched bigint, + OUT n_blocks_hit bigint, + OUT db_time bigint, + OUT cpu_time bigint, + OUT execution_time bigint, + OUT parse_time bigint, + OUT plan_time bigint, + OUT rewrite_time bigint, + OUT pl_execution_time bigint, + OUT pl_compilation_time bigint, + OUT data_io_time bigint, + OUT net_send_info text, + OUT net_recv_info text, + OUT net_stream_send_info text, + OUT net_stream_recv_info text, + OUT lock_count bigint, + OUT lock_time bigint, + OUT lock_wait_count bigint, + OUT lock_wait_time bigint, + OUT lock_max_count bigint, + OUT lwlock_count bigint, + OUT lwlock_wait_count bigint, + OUT lwlock_time bigint, + OUT lwlock_wait_time bigint, + OUT details bytea, + OUT is_slow_sql boolean, + OUT trace_id text, + OUT advise text) +RETURNS SETOF record NOT FENCED NOT SHIPPABLE ROWS 10000 +LANGUAGE internal AS $function$standby_statement_history_1v$function$; + +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 3119; +CREATE FUNCTION dbe_perf.standby_statement_history( + IN only_slow boolean, + VARIADIC finish_time timestamp with time zone[], + OUT db_name name, + OUT schema_name name, + OUT origin_node integer, + OUT user_name name, + OUT application_name text, + OUT client_addr text, + OUT client_port integer, + OUT unique_query_id bigint, + OUT debug_query_id bigint, + OUT query text, + OUT start_time timestamp with time zone, + OUT finish_time timestamp with time zone, + OUT slow_sql_threshold bigint, + OUT transaction_id bigint, + OUT thread_id bigint, + OUT session_id bigint, + OUT n_soft_parse bigint, + OUT n_hard_parse bigint, + OUT query_plan text, + OUT n_returned_rows bigint, + OUT n_tuples_fetched bigint, + OUT n_tuples_returned bigint, + OUT n_tuples_inserted bigint, + OUT n_tuples_updated bigint, + OUT n_tuples_deleted bigint, + OUT n_blocks_fetched bigint, + OUT n_blocks_hit bigint, + OUT db_time bigint, + OUT cpu_time bigint, + OUT execution_time bigint, + OUT parse_time bigint, + OUT plan_time bigint, + OUT rewrite_time bigint, + OUT pl_execution_time bigint, + OUT pl_compilation_time bigint, + OUT data_io_time bigint, + OUT net_send_info text, + OUT net_recv_info text, + OUT net_stream_send_info text, + OUT net_stream_recv_info text, + OUT lock_count bigint, + OUT lock_time bigint, + OUT lock_wait_count bigint, + OUT lock_wait_time bigint, + OUT lock_max_count bigint, + OUT lwlock_count bigint, + OUT lwlock_wait_count bigint, + OUT lwlock_time bigint, + OUT lwlock_wait_time bigint, + OUT details bytea, + OUT is_slow_sql boolean, + OUT trace_id text, + OUT advise text) +RETURNS SETOF record NOT FENCED NOT SHIPPABLE ROWS 10000 +LANGUAGE internal AS $function$standby_statement_history$function$; + +CREATE VIEW DBE_PERF.statement AS + SELECT * FROM get_instr_unique_sql(); + +DROP FUNCTION IF EXISTS dbe_perf.get_summary_statement() cascade; +CREATE OR REPLACE FUNCTION dbe_perf.get_summary_statement() +RETURNS setof dbe_perf.statement +AS $$ +DECLARE + row_data dbe_perf.statement%rowtype; + row_name record; + query_str text; + query_str_nodes text; + BEGIN + --Get all the node names + query_str_nodes := 'select * from dbe_perf.node_name'; + FOR row_name IN EXECUTE(query_str_nodes) LOOP + query_str := 'SELECT * FROM dbe_perf.statement'; + FOR row_data IN EXECUTE(query_str) LOOP + return next row_data; + END LOOP; + END LOOP; + return; + END; $$ +LANGUAGE 'plpgsql' NOT FENCED; + +CREATE VIEW DBE_PERF.summary_statement AS + SELECT * FROM DBE_PERF.get_summary_statement(); + +CREATE VIEW DBE_PERF.statement_history AS + select * from pg_catalog.statement_history; + +DROP FUNCTION IF EXISTS dbe_perf.get_global_full_sql_by_timestamp(timestamp with time zone, timestamp with time zone); +CREATE OR REPLACE FUNCTION dbe_perf.get_global_full_sql_by_timestamp + (in start_timestamp timestamp with time zone, + in end_timestamp timestamp with time zone, + OUT node_name name, + OUT db_name name, + OUT schema_name name, + OUT origin_node integer, + OUT user_name name, + OUT application_name text, + OUT client_addr text, + OUT client_port integer, + OUT unique_query_id bigint, + OUT debug_query_id bigint, + OUT query text, + OUT start_time timestamp with time zone, + OUT finish_time timestamp with time zone, + OUT slow_sql_threshold bigint, + OUT transaction_id bigint, + OUT thread_id bigint, + OUT session_id bigint, + OUT n_soft_parse bigint, + OUT n_hard_parse bigint, + OUT query_plan text, + OUT n_returned_rows bigint, + OUT n_tuples_fetched bigint, + OUT n_tuples_returned bigint, + OUT n_tuples_inserted bigint, + OUT n_tuples_updated bigint, + OUT n_tuples_deleted bigint, + OUT n_blocks_fetched bigint, + OUT n_blocks_hit bigint, + OUT db_time bigint, + OUT cpu_time bigint, + OUT execution_time bigint, + OUT parse_time bigint, + OUT plan_time bigint, + OUT rewrite_time bigint, + OUT pl_execution_time bigint, + OUT pl_compilation_time bigint, + OUT data_io_time bigint, + OUT net_send_info text, + OUT net_recv_info text, + OUT net_stream_send_info text, + OUT net_stream_recv_info text, + OUT lock_count bigint, + OUT lock_time bigint, + OUT lock_wait_count bigint, + OUT lock_wait_time bigint, + OUT lock_max_count bigint, + OUT lwlock_count bigint, + OUT lwlock_wait_count bigint, + OUT lwlock_time bigint, + OUT lwlock_wait_time bigint, + OUT details bytea, + OUT is_slow_sql bool, + OUT trace_id text, + OUT advise text + ) + RETURNS setof record + AS $$ + DECLARE + row_data pg_catalog.statement_history%rowtype; + row_name record; + query_str text; + -- node name + query_str_nodes text; + BEGIN + -- Get all node names(CN + master DN) + query_str_nodes := 'select * from dbe_perf.node_name'; + FOR row_name IN EXECUTE(query_str_nodes) LOOP + query_str := 'SELECT * FROM DBE_PERF.statement_history where start_time >= ''' ||$1|| ''' and start_time <= ''' || $2 || ''''; + FOR row_data IN EXECUTE(query_str) LOOP + node_name := row_name.node_name; + db_name := row_data.db_name; + schema_name := row_data.schema_name; + origin_node := row_data.origin_node; + user_name := row_data.user_name; + application_name := row_data.application_name; + client_addr := row_data.client_addr; + client_port := row_data.client_port; + unique_query_id := row_data.unique_query_id; + debug_query_id := row_data.debug_query_id; + query := row_data.query; + start_time := row_data.start_time; + finish_time := row_data.finish_time; + slow_sql_threshold := row_data.slow_sql_threshold; + transaction_id := row_data.transaction_id; + thread_id := row_data.thread_id; + session_id := row_data.session_id; + n_soft_parse := row_data.n_soft_parse; + n_hard_parse := row_data.n_hard_parse; + query_plan := row_data.query_plan; + n_returned_rows := row_data.n_returned_rows; + n_tuples_fetched := row_data.n_tuples_fetched; + n_tuples_returned := row_data.n_tuples_returned; + n_tuples_inserted := row_data.n_tuples_inserted; + n_tuples_updated := row_data.n_tuples_updated; + n_tuples_deleted := row_data.n_tuples_deleted; + n_blocks_fetched := row_data.n_blocks_fetched; + n_blocks_hit := row_data.n_blocks_hit; + db_time := row_data.db_time; + cpu_time := row_data.cpu_time; + execution_time := row_data.execution_time; + parse_time := row_data.parse_time; + plan_time := row_data.plan_time; + rewrite_time := row_data.rewrite_time; + pl_execution_time := row_data.pl_execution_time; + pl_compilation_time := row_data.pl_compilation_time; + data_io_time := row_data.data_io_time; + net_send_info := row_data.net_send_info; + net_recv_info := row_data.net_recv_info; + net_stream_send_info := row_data.net_stream_send_info; + net_stream_recv_info := row_data.net_stream_recv_info; + lock_count := row_data.lock_count; + lock_time := row_data.lock_time; + lock_wait_count := row_data.lock_wait_count; + lock_wait_time := row_data.lock_wait_time; + lock_max_count := row_data.lock_max_count; + lwlock_count := row_data.lwlock_count; + lwlock_wait_count := row_data.lwlock_wait_count; + lwlock_time := row_data.lwlock_time; + lwlock_wait_time := row_data.lwlock_wait_time; + details := row_data.details; + is_slow_sql := row_data.is_slow_sql; + trace_id := row_data.trace_id; + advise := row_data.advise; + return next; + END LOOP; + END LOOP; + return; + END; $$ +LANGUAGE 'plpgsql' NOT FENCED; + +DROP FUNCTION IF EXISTS dbe_perf.get_global_slow_sql_by_timestamp(timestamp with time zone, timestamp with time zone); +CREATE OR REPLACE FUNCTION DBE_PERF.get_global_slow_sql_by_timestamp + (in start_timestamp timestamp with time zone, + in end_timestamp timestamp with time zone, + OUT node_name name, + OUT db_name name, + OUT schema_name name, + OUT origin_node integer, + OUT user_name name, + OUT application_name text, + OUT client_addr text, + OUT client_port integer, + OUT unique_query_id bigint, + OUT debug_query_id bigint, + OUT query text, + OUT start_time timestamp with time zone, + OUT finish_time timestamp with time zone, + OUT slow_sql_threshold bigint, + OUT transaction_id bigint, + OUT thread_id bigint, + OUT session_id bigint, + OUT n_soft_parse bigint, + OUT n_hard_parse bigint, + OUT query_plan text, + OUT n_returned_rows bigint, + OUT n_tuples_fetched bigint, + OUT n_tuples_returned bigint, + OUT n_tuples_inserted bigint, + OUT n_tuples_updated bigint, + OUT n_tuples_deleted bigint, + OUT n_blocks_fetched bigint, + OUT n_blocks_hit bigint, + OUT db_time bigint, + OUT cpu_time bigint, + OUT execution_time bigint, + OUT parse_time bigint, + OUT plan_time bigint, + OUT rewrite_time bigint, + OUT pl_execution_time bigint, + OUT pl_compilation_time bigint, + OUT data_io_time bigint, + OUT net_send_info text, + OUT net_recv_info text, + OUT net_stream_send_info text, + OUT net_stream_recv_info text, + OUT lock_count bigint, + OUT lock_time bigint, + OUT lock_wait_count bigint, + OUT lock_wait_time bigint, + OUT lock_max_count bigint, + OUT lwlock_count bigint, + OUT lwlock_wait_count bigint, + OUT lwlock_time bigint, + OUT lwlock_wait_time bigint, + OUT details bytea, + OUT is_slow_sql bool, + OUT trace_id text, + OUT advise text) + RETURNS setof record + AS $$ + DECLARE + row_data pg_catalog.statement_history%rowtype; + row_name record; + query_str text; + -- node name + query_str_nodes text; + BEGIN + -- Get all node names(CN + master DN) + query_str_nodes := 'select * from dbe_perf.node_name'; + FOR row_name IN EXECUTE(query_str_nodes) LOOP + query_str := 'SELECT * FROM DBE_PERF.statement_history where start_time >= ''' ||$1|| ''' and start_time <= ''' || $2 || ''' and is_slow_sql = true '; + FOR row_data IN EXECUTE(query_str) LOOP + node_name := row_name.node_name; + db_name := row_data.db_name; + schema_name := row_data.schema_name; + origin_node := row_data.origin_node; + user_name := row_data.user_name; + application_name := row_data.application_name; + client_addr := row_data.client_addr; + client_port := row_data.client_port; + unique_query_id := row_data.unique_query_id; + debug_query_id := row_data.debug_query_id; + query := row_data.query; + start_time := row_data.start_time; + finish_time := row_data.finish_time; + slow_sql_threshold := row_data.slow_sql_threshold; + transaction_id := row_data.transaction_id; + thread_id := row_data.thread_id; + session_id := row_data.session_id; + n_soft_parse := row_data.n_soft_parse; + n_hard_parse := row_data.n_hard_parse; + query_plan := row_data.query_plan; + n_returned_rows := row_data.n_returned_rows; + n_tuples_fetched := row_data.n_tuples_fetched; + n_tuples_returned := row_data.n_tuples_returned; + n_tuples_inserted := row_data.n_tuples_inserted; + n_tuples_updated := row_data.n_tuples_updated; + n_tuples_deleted := row_data.n_tuples_deleted; + n_blocks_fetched := row_data.n_blocks_fetched; + n_blocks_hit := row_data.n_blocks_hit; + db_time := row_data.db_time; + cpu_time := row_data.cpu_time; + execution_time := row_data.execution_time; + parse_time := row_data.parse_time; + plan_time := row_data.plan_time; + rewrite_time := row_data.rewrite_time; + pl_execution_time := row_data.pl_execution_time; + pl_compilation_time := row_data.pl_compilation_time; + data_io_time := row_data.data_io_time; + net_send_info := row_data.net_send_info; + net_recv_info := row_data.net_recv_info; + net_stream_send_info := row_data.net_stream_send_info; + net_stream_recv_info := row_data.net_stream_recv_info; + lock_count := row_data.lock_count; + lock_time := row_data.lock_time; + lock_wait_count := row_data.lock_wait_count; + lock_wait_time := row_data.lock_wait_time; + lock_max_count := row_data.lock_max_count; + lwlock_count := row_data.lwlock_count; + lwlock_wait_count := row_data.lwlock_wait_count; + lwlock_time := row_data.lwlock_time; + lwlock_wait_time := row_data.lwlock_wait_time; + details := row_data.details; + is_slow_sql := row_data.is_slow_sql; + trace_id := row_data.trace_id; + advise := row_data.advise; + return next; + END LOOP; + END LOOP; + return; + END; $$ +LANGUAGE 'plpgsql' NOT FENCED; + +/* we process wlm releates.*/ +DROP VIEW IF EXISTS pg_catalog.gs_wlm_session_info_all CASCADE; +DROP VIEW IF EXISTS pg_catalog.gs_wlm_session_info CASCADE; +DROP VIEW IF EXISTS pg_catalog.gs_wlm_session_info CASCADE; +DROP FUNCTION IF EXISTS pg_catalog.create_wlm_session_info(IN flag int) CASCADE; +DROP FUNCTION IF EXISTS pg_catalog.pg_stat_get_wlm_session_info(OID) cascade; + +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 5002; +CREATE OR REPLACE FUNCTION pg_catalog.pg_stat_get_wlm_session_info +(OID, + OUT datid oid, + OUT dbname text, + OUT schemaname text, + OUT nodename text, + OUT username text, + OUT application_name text, + OUT client_addr inet, + OUT client_hostname text, + OUT client_port integer, + OUT query_band text, + OUT block_time bigint, + OUT start_time timestamp with time zone, + OUT finish_time timestamp with time zone, + OUT duration bigint, + OUT estimate_total_time bigint, + OUT status text, + OUT abort_info text, + OUT resource_pool text, + OUT control_group text, + OUT estimate_memory integer, + OUT min_peak_memory integer, + OUT max_peak_memory integer, + OUT average_peak_memory integer, + OUT memory_skew_percent integer, + OUT spill_info text, + OUT min_spill_size integer, + OUT max_spill_size integer, + OUT average_spill_size integer, + OUT spill_skew_percent integer, + OUT min_dn_time bigint, + OUT max_dn_time bigint, + OUT average_dn_time bigint, + OUT dntime_skew_percent integer, + OUT min_cpu_time bigint, + OUT max_cpu_time bigint, + OUT total_cpu_time bigint, + OUT cpu_skew_percent integer, + OUT min_peak_iops integer, + OUT max_peak_iops integer, + OUT average_peak_iops integer, + OUT iops_skew_percent integer, + OUT warning text, + OUT queryid bigint, + OUT query text, + OUT query_plan text, + OUT node_group text, + OUT cpu_top1_node_name text, + OUT cpu_top2_node_name text, + OUT cpu_top3_node_name text, + OUT cpu_top4_node_name text, + OUT cpu_top5_node_name text, + OUT mem_top1_node_name text, + OUT mem_top2_node_name text, + OUT mem_top3_node_name text, + OUT mem_top4_node_name text, + OUT mem_top5_node_name text, + OUT cpu_top1_value bigint, + OUT cpu_top2_value bigint, + OUT cpu_top3_value bigint, + OUT cpu_top4_value bigint, + OUT cpu_top5_value bigint, + OUT mem_top1_value bigint, + OUT mem_top2_value bigint, + OUT mem_top3_value bigint, + OUT mem_top4_value bigint, + OUT mem_top5_value bigint, + OUT top_mem_dn text, + OUT top_cpu_dn text, + OUT n_returned_rows bigint, + OUT n_tuples_fetched bigint, + OUT n_tuples_returned bigint, + OUT n_tuples_inserted bigint, + OUT n_tuples_updated bigint, + OUT n_tuples_deleted bigint, + OUT n_blocks_fetched bigint, + OUT n_blocks_hit bigint, + OUT db_time bigint, + OUT cpu_time bigint, + OUT execution_time bigint, + OUT parse_time bigint, + OUT plan_time bigint, + OUT rewrite_time bigint, + OUT pl_execution_time bigint, + OUT pl_compilation_time bigint, + OUT net_send_time bigint, + OUT data_io_time bigint, + OUT is_slow_query bigint + ) RETURNS setof record LANGUAGE INTERNAL VOLATILE NOT FENCED as 'pg_stat_get_wlm_session_info'; + +CREATE VIEW pg_catalog.gs_wlm_session_info_all AS +SELECT * FROM pg_catalog.pg_stat_get_wlm_session_info(0); + +--process wlm_session info +CREATE VIEW pg_catalog.gs_wlm_session_info AS +SELECT + S.datid, + S.dbname, + S.schemaname, + S.nodename, + S.username, + S.application_name, + S.client_addr, + S.client_hostname, + S.client_port, + S.query_band, + S.block_time, + S.start_time, + S.finish_time, + S.duration, + S.estimate_total_time, + S.status, + S.abort_info, + S.resource_pool, + S.control_group, + S.estimate_memory, + S.min_peak_memory, + S.max_peak_memory, + S.average_peak_memory, + S.memory_skew_percent, + S.spill_info, + S.min_spill_size, + S.max_spill_size, + S.average_spill_size, + S.spill_skew_percent, + S.min_dn_time, + S.max_dn_time, + S.average_dn_time, + S.dntime_skew_percent, + S.min_cpu_time, + S.max_cpu_time, + S.total_cpu_time, + S.cpu_skew_percent, + S.min_peak_iops, + S.max_peak_iops, + S.average_peak_iops, + S.iops_skew_percent, + S.warning, + S.queryid, + S.query, + S.query_plan, + S.node_group, + S.cpu_top1_node_name, + S.cpu_top2_node_name, + S.cpu_top3_node_name, + S.cpu_top4_node_name, + S.cpu_top5_node_name, + S.mem_top1_node_name, + S.mem_top2_node_name, + S.mem_top3_node_name, + S.mem_top4_node_name, + S.mem_top5_node_name, + S.cpu_top1_value, + S.cpu_top2_value, + S.cpu_top3_value, + S.cpu_top4_value, + S.cpu_top5_value, + S.mem_top1_value, + S.mem_top2_value, + S.mem_top3_value, + S.mem_top4_value, + S.mem_top5_value, + S.top_mem_dn, + S.top_cpu_dn +FROM gs_wlm_session_query_info_all S; + +CREATE VIEW pg_catalog.gs_wlm_session_history AS +SELECT + S.datid, + S.dbname, + S.schemaname, + S.nodename, + S.username, + S.application_name, + S.client_addr, + S.client_hostname, + S.client_port, + S.query_band, + S.block_time, + S.start_time, + S.finish_time, + S.duration, + S.estimate_total_time, + S.status, + S.abort_info, + S.resource_pool, + S.control_group, + S.estimate_memory, + S.min_peak_memory, + S.max_peak_memory, + S.average_peak_memory, + S.memory_skew_percent, + S.spill_info, + S.min_spill_size, + S.max_spill_size, + S.average_spill_size, + S.spill_skew_percent, + S.min_dn_time, + S.max_dn_time, + S.average_dn_time, + S.dntime_skew_percent, + S.min_cpu_time, + S.max_cpu_time, + S.total_cpu_time, + S.cpu_skew_percent, + S.min_peak_iops, + S.max_peak_iops, + S.average_peak_iops, + S.iops_skew_percent, + S.warning, + S.queryid, + S.query, + S.query_plan, + S.node_group, + S.cpu_top1_node_name, + S.cpu_top2_node_name, + S.cpu_top3_node_name, + S.cpu_top4_node_name, + S.cpu_top5_node_name, + S.mem_top1_node_name, + S.mem_top2_node_name, + S.mem_top3_node_name, + S.mem_top4_node_name, + S.mem_top5_node_name, + S.cpu_top1_value, + S.cpu_top2_value, + S.cpu_top3_value, + S.cpu_top4_value, + S.cpu_top5_value, + S.mem_top1_value, + S.mem_top2_value, + S.mem_top3_value, + S.mem_top4_value, + S.mem_top5_value, + S.top_mem_dn, + S.top_cpu_dn +FROM pg_catalog.gs_wlm_session_info_all S; + +CREATE OR REPLACE FUNCTION pg_catalog.create_wlm_session_info(IN flag int) +RETURNS int +AS $$ +DECLARE + query_str text; + record_cnt int; + BEGIN + record_cnt := 0; + + query_str := 'SELECT * FROM pg_catalog.pg_stat_get_wlm_session_info(1)'; + + IF flag > 0 THEN + EXECUTE 'INSERT INTO pg_catalog.gs_wlm_session_query_info_all ' || query_str; + ELSE + EXECUTE query_str; + END IF; + + RETURN record_cnt; + END; $$ +LANGUAGE plpgsql NOT FENCED; + +DROP FUNCTION IF EXISTS dbe_perf.global_slow_query_info() cascade; +DROP FUNCTION IF EXISTS dbe_perf.global_slow_query_history() cascade; +DROP VIEW IF EXISTS dbe_perf.gs_slow_query_info cascade; +DROP VIEW IF EXISTS dbe_perf.gs_slow_query_history cascade; + +CREATE OR REPLACE VIEW dbe_perf.gs_slow_query_info AS +SELECT + S.dbname, + S.schemaname, + S.nodename, + S.username, + S.queryid, + S.query, + S.start_time, + S.finish_time, + S.duration, + S.query_plan, + S.n_returned_rows, + S.n_tuples_fetched, + S.n_tuples_returned, + S.n_tuples_inserted, + S.n_tuples_updated, + S.n_tuples_deleted, + S.n_blocks_fetched, + S.n_blocks_hit, + S.db_time, + S.cpu_time, + S.execution_time, + S.parse_time, + S.plan_time, + S.rewrite_time, + S.pl_execution_time, + S.pl_compilation_time, + S.net_send_time, + S.data_io_time +FROM gs_wlm_session_query_info_all S where S.is_slow_query = 1; + +CREATE OR REPLACE VIEW dbe_perf.gs_slow_query_history AS +SELECT + S.dbname, + S.schemaname, + S.nodename, + S.username, + S.queryid, + S.query, + S.start_time, + S.finish_time, + S.duration, + S.query_plan, + S.n_returned_rows, + S.n_tuples_fetched, + S.n_tuples_returned, + S.n_tuples_inserted, + S.n_tuples_updated, + S.n_tuples_deleted, + S.n_blocks_fetched, + S.n_blocks_hit, + S.db_time, + S.cpu_time, + S.execution_time, + S.parse_time, + S.plan_time, + S.rewrite_time, + S.pl_execution_time, + S.pl_compilation_time, + S.net_send_time, + S.data_io_time +FROM pg_catalog.pg_stat_get_wlm_session_info(0) S where S.is_slow_query = 1; + +CREATE OR REPLACE FUNCTION dbe_perf.global_slow_query_history +RETURNS setof dbe_perf.gs_slow_query_history +AS $$ +DECLARE + row_data dbe_perf.gs_slow_query_history%rowtype; + row_name record; + query_str text; + query_str_nodes text; + BEGIN + --Get all the node names + query_str_nodes := 'SELECT node_name FROM pgxc_node WHERE node_type=''C'' AND nodeis_active = true'; + FOR row_name IN EXECUTE(query_str_nodes) LOOP + query_str := 'EXECUTE DIRECT ON (' || row_name.node_name || ') ''SELECT * FROM dbe_perf.gs_slow_query_history'''; + FOR row_data IN EXECUTE(query_str) LOOP + return next row_data; + END LOOP; + END LOOP; + return; + END; $$ +LANGUAGE 'plpgsql' NOT FENCED; + +CREATE OR REPLACE FUNCTION dbe_perf.global_slow_query_info() +RETURNS setof dbe_perf.gs_slow_query_info +AS $$ +DECLARE + row_data dbe_perf.gs_slow_query_info%rowtype; + row_name record; + query_str text; + query_str_nodes text; + BEGIN + --Get all the node names + query_str_nodes := 'SELECT node_name FROM pgxc_node WHERE node_type=''C'' AND nodeis_active = true'; + FOR row_name IN EXECUTE(query_str_nodes) LOOP + query_str := 'EXECUTE DIRECT ON (' || row_name.node_name || ') ''SELECT * FROM dbe_perf.gs_slow_query_info'''; + FOR row_data IN EXECUTE(query_str) LOOP + return next row_data; + END LOOP; + END LOOP; + return; + END; $$ +LANGUAGE 'plpgsql' NOT FENCED; + +CREATE OR REPLACE VIEW DBE_PERF.global_slow_query_history AS +SELECT * FROM DBE_PERF.global_slow_query_history(); + +CREATE OR REPLACE VIEW DBE_PERF.global_slow_query_info AS +SELECT * FROM DBE_PERF.global_slow_query_info(); + +GRANT SELECT ON TABLE pg_catalog.gs_wlm_session_query_info_all TO PUBLIC; +GRANT SELECT ON TABLE pg_catalog.gs_wlm_session_history TO PUBLIC; +GRANT SELECT ON TABLE pg_catalog.gs_wlm_session_info TO PUBLIC; +GRANT SELECT ON TABLE pg_catalog.gs_wlm_session_info_all TO PUBLIC; +GRANT SELECT ON TABLE DBE_PERF.STATEMENT TO PUBLIC; +GRANT SELECT ON TABLE DBE_PERF.summary_statement TO PUBLIC; +GRANT SELECT ON TABLE DBE_PERF.statement_history TO PUBLIC; +GRANT SELECT ON TABLE DBE_PERF.global_slow_query_history TO PUBLIC; +GRANT SELECT ON TABLE DBE_PERF.global_slow_query_info TO PUBLIC; diff --git a/src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback_catalog_maindb_92_906.sql b/src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback_catalog_maindb_92_906.sql new file mode 100644 index 000000000..c671518ec --- /dev/null +++ b/src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback_catalog_maindb_92_906.sql @@ -0,0 +1,22 @@ +-- this table will be remove column +-- openGauss=# select table_schema, table_name,column_name from information_schema.columns where column_name = 'db_time'; +-- table_schema | table_name | column_name +-- --------------+-------------------------------+------------- +-- pg_catalog | gs_wlm_session_query_info_all | db_time --table +-- pg_catalog | statement_history | db_time --unlogged table, update in post-upgrade + +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all drop column if exists srt1_q cascade; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all drop column if exists srt2_simple_query cascade ; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all drop column if exists srt3_analyze_rewrite cascade ; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all drop column if exists srt4_plan_query cascade ; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all drop column if exists srt5_light_query cascade ; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all drop column if exists srt6_p cascade ; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all drop column if exists srt7_b cascade ; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all drop column if exists srt8_e cascade ; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all drop column if exists srt9_d cascade ; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all drop column if exists srt10_s cascade ; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all drop column if exists srt11_c cascade ; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all drop column if exists srt12_u cascade ; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all drop column if exists srt13_before_query cascade ; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all drop column if exists srt14_after_query cascade ; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all drop column if exists rtt_unknown cascade ; diff --git a/src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback-post_catalog_otherdb_92_906.sql b/src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback-post_catalog_otherdb_92_906.sql index d1f556cdc..f2a9f31ac 100644 --- a/src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback-post_catalog_otherdb_92_906.sql +++ b/src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback-post_catalog_otherdb_92_906.sql @@ -1 +1,957 @@ DROP FUNCTION IF EXISTS pg_catalog.ss_txnstatus_cache_stat() CASCADE; + +-- those type of view and producer will recreate. +-- openGauss=# select table_schema, table_name,column_name from information_schema.columns where column_name = 'db_time'; +-- table_schema | table_name | column_name +-- --------------+-------------------------------+------------- +-- pg_catalog | gs_wlm_session_query_info_all | db_time -- table +-- pg_catalog | gs_wlm_session_info_all | db_time -- view +-- dbe_perf | statement | db_time -- view +-- pg_catalog | statement_history | db_time -- unlogged table +-- dbe_perf | statement_history | db_time -- view +-- dbe_perf | summary_statement | db_time -- view +-- dbe_perf | gs_slow_query_info | db_time -- view +-- dbe_perf | gs_slow_query_history | db_time -- view +-- dbe_perf | global_slow_query_history | db_time -- view +-- dbe_perf | global_slow_query_info | db_time -- view +-- those proc will recreated. +-- openGauss=# select proname, pronamespace from pg_proc where proargnames @> array['db_time']; +-- proname | pronamespace +-- ----------------------------------+-------------- +-- get_instr_unique_sql | 11 +-- pg_stat_get_wlm_session_info | 11 +-- standby_statement_history | 4988 +-- standby_statement_history | 4988 +-- get_global_full_sql_by_timestamp | 4988 +-- get_global_slow_sql_by_timestamp | 4988 + +/* We process history related tables、views and function now. */ +DROP VIEW IF EXISTS DBE_PERF.statement CASCADE; +DROP VIEW IF EXISTS DBE_PERF.summary_statement CASCADE; +DROP VIEW IF EXISTS DBE_PERF.statement_history CASCADE; +DROP FUNCTION IF EXISTS pg_catalog.get_instr_unique_sql() CASCADE; +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 5702; +CREATE FUNCTION pg_catalog.get_instr_unique_sql +( + OUT node_name name, + OUT node_id integer, + OUT user_name name, + OUT user_id oid, + OUT unique_sql_id bigint, + OUT query text, + OUT n_calls bigint, + OUT min_elapse_time bigint, + OUT max_elapse_time bigint, + OUT total_elapse_time bigint, + OUT n_returned_rows bigint, + OUT n_tuples_fetched bigint, + OUT n_tuples_returned bigint, + OUT n_tuples_inserted bigint, + OUT n_tuples_updated bigint, + OUT n_tuples_deleted bigint, + OUT n_blocks_fetched bigint, + OUT n_blocks_hit bigint, + OUT n_soft_parse bigint, + OUT n_hard_parse bigint, + OUT db_time bigint, + OUT cpu_time bigint, + OUT execution_time bigint, + OUT parse_time bigint, + OUT plan_time bigint, + OUT rewrite_time bigint, + OUT pl_execution_time bigint, + OUT pl_compilation_time bigint, + OUT data_io_time bigint, + OUT net_send_info text, + Out net_recv_info text, + OUT net_stream_send_info text, + OUT net_stream_recv_info text, + OUT last_updated timestamp with time zone, + OUT sort_count bigint, + OUT sort_time bigint, + OUT sort_mem_used bigint, + OUT sort_spill_count bigint, + OUT sort_spill_size bigint, + OUT hash_count bigint, + OUT hash_time bigint, + OUT hash_mem_used bigint, + OUT hash_spill_count bigint, + OUT hash_spill_size bigint +) +RETURNS setof record LANGUAGE INTERNAL VOLATILE NOT FENCED as 'get_instr_unique_sql'; + +DROP INDEX IF EXISTS pg_catalog.statement_history_time_idx; +DROP TABLE IF EXISTS pg_catalog.statement_history cascade; + +CREATE unlogged table IF NOT EXISTS pg_catalog.statement_history( + db_name name, + schema_name name, + origin_node integer, + user_name name, + application_name text, + client_addr text, + client_port integer, + unique_query_id bigint, + debug_query_id bigint, + query text, + start_time timestamp with time zone, + finish_time timestamp with time zone, + slow_sql_threshold bigint, + transaction_id bigint, + thread_id bigint, + session_id bigint, + n_soft_parse bigint, + n_hard_parse bigint, + query_plan text, + n_returned_rows bigint, + n_tuples_fetched bigint, + n_tuples_returned bigint, + n_tuples_inserted bigint, + n_tuples_updated bigint, + n_tuples_deleted bigint, + n_blocks_fetched bigint, + n_blocks_hit bigint, + db_time bigint, + cpu_time bigint, + execution_time bigint, + parse_time bigint, + plan_time bigint, + rewrite_time bigint, + pl_execution_time bigint, + pl_compilation_time bigint, + data_io_time bigint, + net_send_info text, + net_recv_info text, + net_stream_send_info text, + net_stream_recv_info text, + lock_count bigint, + lock_time bigint, + lock_wait_count bigint, + lock_wait_time bigint, + lock_max_count bigint, + lwlock_count bigint, + lwlock_wait_count bigint, + lwlock_time bigint, + lwlock_wait_time bigint, + details bytea, + is_slow_sql boolean, + trace_id text, + advise text +); +REVOKE ALL on table pg_catalog.statement_history FROM public; +create index pg_catalog.statement_history_time_idx on pg_catalog.statement_history USING btree (start_time, is_slow_sql); + +DROP FUNCTION IF EXISTS dbe_perf.standby_statement_history(boolean); +DROP FUNCTION IF EXISTS dbe_perf.standby_statement_history(boolean, timestamp with time zone[]); +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 3118; +CREATE FUNCTION dbe_perf.standby_statement_history( + IN only_slow boolean, + OUT db_name name, + OUT schema_name name, + OUT origin_node integer, + OUT user_name name, + OUT application_name text, + OUT client_addr text, + OUT client_port integer, + OUT unique_query_id bigint, + OUT debug_query_id bigint, + OUT query text, + OUT start_time timestamp with time zone, + OUT finish_time timestamp with time zone, + OUT slow_sql_threshold bigint, + OUT transaction_id bigint, + OUT thread_id bigint, + OUT session_id bigint, + OUT n_soft_parse bigint, + OUT n_hard_parse bigint, + OUT query_plan text, + OUT n_returned_rows bigint, + OUT n_tuples_fetched bigint, + OUT n_tuples_returned bigint, + OUT n_tuples_inserted bigint, + OUT n_tuples_updated bigint, + OUT n_tuples_deleted bigint, + OUT n_blocks_fetched bigint, + OUT n_blocks_hit bigint, + OUT db_time bigint, + OUT cpu_time bigint, + OUT execution_time bigint, + OUT parse_time bigint, + OUT plan_time bigint, + OUT rewrite_time bigint, + OUT pl_execution_time bigint, + OUT pl_compilation_time bigint, + OUT data_io_time bigint, + OUT net_send_info text, + OUT net_recv_info text, + OUT net_stream_send_info text, + OUT net_stream_recv_info text, + OUT lock_count bigint, + OUT lock_time bigint, + OUT lock_wait_count bigint, + OUT lock_wait_time bigint, + OUT lock_max_count bigint, + OUT lwlock_count bigint, + OUT lwlock_wait_count bigint, + OUT lwlock_time bigint, + OUT lwlock_wait_time bigint, + OUT details bytea, + OUT is_slow_sql boolean, + OUT trace_id text, + OUT advise text) +RETURNS SETOF record NOT FENCED NOT SHIPPABLE ROWS 10000 +LANGUAGE internal AS $function$standby_statement_history_1v$function$; + +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 3119; +CREATE FUNCTION dbe_perf.standby_statement_history( + IN only_slow boolean, + VARIADIC finish_time timestamp with time zone[], + OUT db_name name, + OUT schema_name name, + OUT origin_node integer, + OUT user_name name, + OUT application_name text, + OUT client_addr text, + OUT client_port integer, + OUT unique_query_id bigint, + OUT debug_query_id bigint, + OUT query text, + OUT start_time timestamp with time zone, + OUT finish_time timestamp with time zone, + OUT slow_sql_threshold bigint, + OUT transaction_id bigint, + OUT thread_id bigint, + OUT session_id bigint, + OUT n_soft_parse bigint, + OUT n_hard_parse bigint, + OUT query_plan text, + OUT n_returned_rows bigint, + OUT n_tuples_fetched bigint, + OUT n_tuples_returned bigint, + OUT n_tuples_inserted bigint, + OUT n_tuples_updated bigint, + OUT n_tuples_deleted bigint, + OUT n_blocks_fetched bigint, + OUT n_blocks_hit bigint, + OUT db_time bigint, + OUT cpu_time bigint, + OUT execution_time bigint, + OUT parse_time bigint, + OUT plan_time bigint, + OUT rewrite_time bigint, + OUT pl_execution_time bigint, + OUT pl_compilation_time bigint, + OUT data_io_time bigint, + OUT net_send_info text, + OUT net_recv_info text, + OUT net_stream_send_info text, + OUT net_stream_recv_info text, + OUT lock_count bigint, + OUT lock_time bigint, + OUT lock_wait_count bigint, + OUT lock_wait_time bigint, + OUT lock_max_count bigint, + OUT lwlock_count bigint, + OUT lwlock_wait_count bigint, + OUT lwlock_time bigint, + OUT lwlock_wait_time bigint, + OUT details bytea, + OUT is_slow_sql boolean, + OUT trace_id text, + OUT advise text) +RETURNS SETOF record NOT FENCED NOT SHIPPABLE ROWS 10000 +LANGUAGE internal AS $function$standby_statement_history$function$; + +CREATE VIEW DBE_PERF.statement AS + SELECT * FROM get_instr_unique_sql(); + +DROP FUNCTION IF EXISTS dbe_perf.get_summary_statement() cascade; +CREATE OR REPLACE FUNCTION dbe_perf.get_summary_statement() +RETURNS setof dbe_perf.statement +AS $$ +DECLARE + row_data dbe_perf.statement%rowtype; + row_name record; + query_str text; + query_str_nodes text; + BEGIN + --Get all the node names + query_str_nodes := 'select * from dbe_perf.node_name'; + FOR row_name IN EXECUTE(query_str_nodes) LOOP + query_str := 'SELECT * FROM dbe_perf.statement'; + FOR row_data IN EXECUTE(query_str) LOOP + return next row_data; + END LOOP; + END LOOP; + return; + END; $$ +LANGUAGE 'plpgsql' NOT FENCED; + +CREATE VIEW DBE_PERF.summary_statement AS + SELECT * FROM DBE_PERF.get_summary_statement(); + +CREATE VIEW DBE_PERF.statement_history AS + select * from pg_catalog.statement_history; + +DROP FUNCTION IF EXISTS dbe_perf.get_global_full_sql_by_timestamp(timestamp with time zone, timestamp with time zone); +CREATE OR REPLACE FUNCTION dbe_perf.get_global_full_sql_by_timestamp + (in start_timestamp timestamp with time zone, + in end_timestamp timestamp with time zone, + OUT node_name name, + OUT db_name name, + OUT schema_name name, + OUT origin_node integer, + OUT user_name name, + OUT application_name text, + OUT client_addr text, + OUT client_port integer, + OUT unique_query_id bigint, + OUT debug_query_id bigint, + OUT query text, + OUT start_time timestamp with time zone, + OUT finish_time timestamp with time zone, + OUT slow_sql_threshold bigint, + OUT transaction_id bigint, + OUT thread_id bigint, + OUT session_id bigint, + OUT n_soft_parse bigint, + OUT n_hard_parse bigint, + OUT query_plan text, + OUT n_returned_rows bigint, + OUT n_tuples_fetched bigint, + OUT n_tuples_returned bigint, + OUT n_tuples_inserted bigint, + OUT n_tuples_updated bigint, + OUT n_tuples_deleted bigint, + OUT n_blocks_fetched bigint, + OUT n_blocks_hit bigint, + OUT db_time bigint, + OUT cpu_time bigint, + OUT execution_time bigint, + OUT parse_time bigint, + OUT plan_time bigint, + OUT rewrite_time bigint, + OUT pl_execution_time bigint, + OUT pl_compilation_time bigint, + OUT data_io_time bigint, + OUT net_send_info text, + OUT net_recv_info text, + OUT net_stream_send_info text, + OUT net_stream_recv_info text, + OUT lock_count bigint, + OUT lock_time bigint, + OUT lock_wait_count bigint, + OUT lock_wait_time bigint, + OUT lock_max_count bigint, + OUT lwlock_count bigint, + OUT lwlock_wait_count bigint, + OUT lwlock_time bigint, + OUT lwlock_wait_time bigint, + OUT details bytea, + OUT is_slow_sql bool, + OUT trace_id text, + OUT advise text + ) + RETURNS setof record + AS $$ + DECLARE + row_data pg_catalog.statement_history%rowtype; + row_name record; + query_str text; + -- node name + query_str_nodes text; + BEGIN + -- Get all node names(CN + master DN) + query_str_nodes := 'select * from dbe_perf.node_name'; + FOR row_name IN EXECUTE(query_str_nodes) LOOP + query_str := 'SELECT * FROM DBE_PERF.statement_history where start_time >= ''' ||$1|| ''' and start_time <= ''' || $2 || ''''; + FOR row_data IN EXECUTE(query_str) LOOP + node_name := row_name.node_name; + db_name := row_data.db_name; + schema_name := row_data.schema_name; + origin_node := row_data.origin_node; + user_name := row_data.user_name; + application_name := row_data.application_name; + client_addr := row_data.client_addr; + client_port := row_data.client_port; + unique_query_id := row_data.unique_query_id; + debug_query_id := row_data.debug_query_id; + query := row_data.query; + start_time := row_data.start_time; + finish_time := row_data.finish_time; + slow_sql_threshold := row_data.slow_sql_threshold; + transaction_id := row_data.transaction_id; + thread_id := row_data.thread_id; + session_id := row_data.session_id; + n_soft_parse := row_data.n_soft_parse; + n_hard_parse := row_data.n_hard_parse; + query_plan := row_data.query_plan; + n_returned_rows := row_data.n_returned_rows; + n_tuples_fetched := row_data.n_tuples_fetched; + n_tuples_returned := row_data.n_tuples_returned; + n_tuples_inserted := row_data.n_tuples_inserted; + n_tuples_updated := row_data.n_tuples_updated; + n_tuples_deleted := row_data.n_tuples_deleted; + n_blocks_fetched := row_data.n_blocks_fetched; + n_blocks_hit := row_data.n_blocks_hit; + db_time := row_data.db_time; + cpu_time := row_data.cpu_time; + execution_time := row_data.execution_time; + parse_time := row_data.parse_time; + plan_time := row_data.plan_time; + rewrite_time := row_data.rewrite_time; + pl_execution_time := row_data.pl_execution_time; + pl_compilation_time := row_data.pl_compilation_time; + data_io_time := row_data.data_io_time; + net_send_info := row_data.net_send_info; + net_recv_info := row_data.net_recv_info; + net_stream_send_info := row_data.net_stream_send_info; + net_stream_recv_info := row_data.net_stream_recv_info; + lock_count := row_data.lock_count; + lock_time := row_data.lock_time; + lock_wait_count := row_data.lock_wait_count; + lock_wait_time := row_data.lock_wait_time; + lock_max_count := row_data.lock_max_count; + lwlock_count := row_data.lwlock_count; + lwlock_wait_count := row_data.lwlock_wait_count; + lwlock_time := row_data.lwlock_time; + lwlock_wait_time := row_data.lwlock_wait_time; + details := row_data.details; + is_slow_sql := row_data.is_slow_sql; + trace_id := row_data.trace_id; + advise := row_data.advise; + return next; + END LOOP; + END LOOP; + return; + END; $$ +LANGUAGE 'plpgsql' NOT FENCED; + +DROP FUNCTION IF EXISTS dbe_perf.get_global_slow_sql_by_timestamp(timestamp with time zone, timestamp with time zone); +CREATE OR REPLACE FUNCTION DBE_PERF.get_global_slow_sql_by_timestamp + (in start_timestamp timestamp with time zone, + in end_timestamp timestamp with time zone, + OUT node_name name, + OUT db_name name, + OUT schema_name name, + OUT origin_node integer, + OUT user_name name, + OUT application_name text, + OUT client_addr text, + OUT client_port integer, + OUT unique_query_id bigint, + OUT debug_query_id bigint, + OUT query text, + OUT start_time timestamp with time zone, + OUT finish_time timestamp with time zone, + OUT slow_sql_threshold bigint, + OUT transaction_id bigint, + OUT thread_id bigint, + OUT session_id bigint, + OUT n_soft_parse bigint, + OUT n_hard_parse bigint, + OUT query_plan text, + OUT n_returned_rows bigint, + OUT n_tuples_fetched bigint, + OUT n_tuples_returned bigint, + OUT n_tuples_inserted bigint, + OUT n_tuples_updated bigint, + OUT n_tuples_deleted bigint, + OUT n_blocks_fetched bigint, + OUT n_blocks_hit bigint, + OUT db_time bigint, + OUT cpu_time bigint, + OUT execution_time bigint, + OUT parse_time bigint, + OUT plan_time bigint, + OUT rewrite_time bigint, + OUT pl_execution_time bigint, + OUT pl_compilation_time bigint, + OUT data_io_time bigint, + OUT net_send_info text, + OUT net_recv_info text, + OUT net_stream_send_info text, + OUT net_stream_recv_info text, + OUT lock_count bigint, + OUT lock_time bigint, + OUT lock_wait_count bigint, + OUT lock_wait_time bigint, + OUT lock_max_count bigint, + OUT lwlock_count bigint, + OUT lwlock_wait_count bigint, + OUT lwlock_time bigint, + OUT lwlock_wait_time bigint, + OUT details bytea, + OUT is_slow_sql bool, + OUT trace_id text, + OUT advise text) + RETURNS setof record + AS $$ + DECLARE + row_data pg_catalog.statement_history%rowtype; + row_name record; + query_str text; + -- node name + query_str_nodes text; + BEGIN + -- Get all node names(CN + master DN) + query_str_nodes := 'select * from dbe_perf.node_name'; + FOR row_name IN EXECUTE(query_str_nodes) LOOP + query_str := 'SELECT * FROM DBE_PERF.statement_history where start_time >= ''' ||$1|| ''' and start_time <= ''' || $2 || ''' and is_slow_sql = true '; + FOR row_data IN EXECUTE(query_str) LOOP + node_name := row_name.node_name; + db_name := row_data.db_name; + schema_name := row_data.schema_name; + origin_node := row_data.origin_node; + user_name := row_data.user_name; + application_name := row_data.application_name; + client_addr := row_data.client_addr; + client_port := row_data.client_port; + unique_query_id := row_data.unique_query_id; + debug_query_id := row_data.debug_query_id; + query := row_data.query; + start_time := row_data.start_time; + finish_time := row_data.finish_time; + slow_sql_threshold := row_data.slow_sql_threshold; + transaction_id := row_data.transaction_id; + thread_id := row_data.thread_id; + session_id := row_data.session_id; + n_soft_parse := row_data.n_soft_parse; + n_hard_parse := row_data.n_hard_parse; + query_plan := row_data.query_plan; + n_returned_rows := row_data.n_returned_rows; + n_tuples_fetched := row_data.n_tuples_fetched; + n_tuples_returned := row_data.n_tuples_returned; + n_tuples_inserted := row_data.n_tuples_inserted; + n_tuples_updated := row_data.n_tuples_updated; + n_tuples_deleted := row_data.n_tuples_deleted; + n_blocks_fetched := row_data.n_blocks_fetched; + n_blocks_hit := row_data.n_blocks_hit; + db_time := row_data.db_time; + cpu_time := row_data.cpu_time; + execution_time := row_data.execution_time; + parse_time := row_data.parse_time; + plan_time := row_data.plan_time; + rewrite_time := row_data.rewrite_time; + pl_execution_time := row_data.pl_execution_time; + pl_compilation_time := row_data.pl_compilation_time; + data_io_time := row_data.data_io_time; + net_send_info := row_data.net_send_info; + net_recv_info := row_data.net_recv_info; + net_stream_send_info := row_data.net_stream_send_info; + net_stream_recv_info := row_data.net_stream_recv_info; + lock_count := row_data.lock_count; + lock_time := row_data.lock_time; + lock_wait_count := row_data.lock_wait_count; + lock_wait_time := row_data.lock_wait_time; + lock_max_count := row_data.lock_max_count; + lwlock_count := row_data.lwlock_count; + lwlock_wait_count := row_data.lwlock_wait_count; + lwlock_time := row_data.lwlock_time; + lwlock_wait_time := row_data.lwlock_wait_time; + details := row_data.details; + is_slow_sql := row_data.is_slow_sql; + trace_id := row_data.trace_id; + advise := row_data.advise; + return next; + END LOOP; + END LOOP; + return; + END; $$ +LANGUAGE 'plpgsql' NOT FENCED; + +/* we process wlm releates.*/ +DROP VIEW IF EXISTS pg_catalog.gs_wlm_session_info_all CASCADE; +DROP VIEW IF EXISTS pg_catalog.gs_wlm_session_info CASCADE; +DROP VIEW IF EXISTS pg_catalog.gs_wlm_session_info CASCADE; +DROP FUNCTION IF EXISTS pg_catalog.create_wlm_session_info(IN flag int) CASCADE; +DROP FUNCTION IF EXISTS pg_catalog.pg_stat_get_wlm_session_info(OID) cascade; + +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 5002; +CREATE OR REPLACE FUNCTION pg_catalog.pg_stat_get_wlm_session_info +(OID, + OUT datid oid, + OUT dbname text, + OUT schemaname text, + OUT nodename text, + OUT username text, + OUT application_name text, + OUT client_addr inet, + OUT client_hostname text, + OUT client_port integer, + OUT query_band text, + OUT block_time bigint, + OUT start_time timestamp with time zone, + OUT finish_time timestamp with time zone, + OUT duration bigint, + OUT estimate_total_time bigint, + OUT status text, + OUT abort_info text, + OUT resource_pool text, + OUT control_group text, + OUT estimate_memory integer, + OUT min_peak_memory integer, + OUT max_peak_memory integer, + OUT average_peak_memory integer, + OUT memory_skew_percent integer, + OUT spill_info text, + OUT min_spill_size integer, + OUT max_spill_size integer, + OUT average_spill_size integer, + OUT spill_skew_percent integer, + OUT min_dn_time bigint, + OUT max_dn_time bigint, + OUT average_dn_time bigint, + OUT dntime_skew_percent integer, + OUT min_cpu_time bigint, + OUT max_cpu_time bigint, + OUT total_cpu_time bigint, + OUT cpu_skew_percent integer, + OUT min_peak_iops integer, + OUT max_peak_iops integer, + OUT average_peak_iops integer, + OUT iops_skew_percent integer, + OUT warning text, + OUT queryid bigint, + OUT query text, + OUT query_plan text, + OUT node_group text, + OUT cpu_top1_node_name text, + OUT cpu_top2_node_name text, + OUT cpu_top3_node_name text, + OUT cpu_top4_node_name text, + OUT cpu_top5_node_name text, + OUT mem_top1_node_name text, + OUT mem_top2_node_name text, + OUT mem_top3_node_name text, + OUT mem_top4_node_name text, + OUT mem_top5_node_name text, + OUT cpu_top1_value bigint, + OUT cpu_top2_value bigint, + OUT cpu_top3_value bigint, + OUT cpu_top4_value bigint, + OUT cpu_top5_value bigint, + OUT mem_top1_value bigint, + OUT mem_top2_value bigint, + OUT mem_top3_value bigint, + OUT mem_top4_value bigint, + OUT mem_top5_value bigint, + OUT top_mem_dn text, + OUT top_cpu_dn text, + OUT n_returned_rows bigint, + OUT n_tuples_fetched bigint, + OUT n_tuples_returned bigint, + OUT n_tuples_inserted bigint, + OUT n_tuples_updated bigint, + OUT n_tuples_deleted bigint, + OUT n_blocks_fetched bigint, + OUT n_blocks_hit bigint, + OUT db_time bigint, + OUT cpu_time bigint, + OUT execution_time bigint, + OUT parse_time bigint, + OUT plan_time bigint, + OUT rewrite_time bigint, + OUT pl_execution_time bigint, + OUT pl_compilation_time bigint, + OUT net_send_time bigint, + OUT data_io_time bigint, + OUT is_slow_query bigint + ) RETURNS setof record LANGUAGE INTERNAL VOLATILE NOT FENCED as 'pg_stat_get_wlm_session_info'; + +CREATE VIEW pg_catalog.gs_wlm_session_info_all AS +SELECT * FROM pg_catalog.pg_stat_get_wlm_session_info(0); + +--process wlm_session info +CREATE VIEW pg_catalog.gs_wlm_session_info AS +SELECT + S.datid, + S.dbname, + S.schemaname, + S.nodename, + S.username, + S.application_name, + S.client_addr, + S.client_hostname, + S.client_port, + S.query_band, + S.block_time, + S.start_time, + S.finish_time, + S.duration, + S.estimate_total_time, + S.status, + S.abort_info, + S.resource_pool, + S.control_group, + S.estimate_memory, + S.min_peak_memory, + S.max_peak_memory, + S.average_peak_memory, + S.memory_skew_percent, + S.spill_info, + S.min_spill_size, + S.max_spill_size, + S.average_spill_size, + S.spill_skew_percent, + S.min_dn_time, + S.max_dn_time, + S.average_dn_time, + S.dntime_skew_percent, + S.min_cpu_time, + S.max_cpu_time, + S.total_cpu_time, + S.cpu_skew_percent, + S.min_peak_iops, + S.max_peak_iops, + S.average_peak_iops, + S.iops_skew_percent, + S.warning, + S.queryid, + S.query, + S.query_plan, + S.node_group, + S.cpu_top1_node_name, + S.cpu_top2_node_name, + S.cpu_top3_node_name, + S.cpu_top4_node_name, + S.cpu_top5_node_name, + S.mem_top1_node_name, + S.mem_top2_node_name, + S.mem_top3_node_name, + S.mem_top4_node_name, + S.mem_top5_node_name, + S.cpu_top1_value, + S.cpu_top2_value, + S.cpu_top3_value, + S.cpu_top4_value, + S.cpu_top5_value, + S.mem_top1_value, + S.mem_top2_value, + S.mem_top3_value, + S.mem_top4_value, + S.mem_top5_value, + S.top_mem_dn, + S.top_cpu_dn +FROM gs_wlm_session_query_info_all S; + +CREATE VIEW pg_catalog.gs_wlm_session_history AS +SELECT + S.datid, + S.dbname, + S.schemaname, + S.nodename, + S.username, + S.application_name, + S.client_addr, + S.client_hostname, + S.client_port, + S.query_band, + S.block_time, + S.start_time, + S.finish_time, + S.duration, + S.estimate_total_time, + S.status, + S.abort_info, + S.resource_pool, + S.control_group, + S.estimate_memory, + S.min_peak_memory, + S.max_peak_memory, + S.average_peak_memory, + S.memory_skew_percent, + S.spill_info, + S.min_spill_size, + S.max_spill_size, + S.average_spill_size, + S.spill_skew_percent, + S.min_dn_time, + S.max_dn_time, + S.average_dn_time, + S.dntime_skew_percent, + S.min_cpu_time, + S.max_cpu_time, + S.total_cpu_time, + S.cpu_skew_percent, + S.min_peak_iops, + S.max_peak_iops, + S.average_peak_iops, + S.iops_skew_percent, + S.warning, + S.queryid, + S.query, + S.query_plan, + S.node_group, + S.cpu_top1_node_name, + S.cpu_top2_node_name, + S.cpu_top3_node_name, + S.cpu_top4_node_name, + S.cpu_top5_node_name, + S.mem_top1_node_name, + S.mem_top2_node_name, + S.mem_top3_node_name, + S.mem_top4_node_name, + S.mem_top5_node_name, + S.cpu_top1_value, + S.cpu_top2_value, + S.cpu_top3_value, + S.cpu_top4_value, + S.cpu_top5_value, + S.mem_top1_value, + S.mem_top2_value, + S.mem_top3_value, + S.mem_top4_value, + S.mem_top5_value, + S.top_mem_dn, + S.top_cpu_dn +FROM pg_catalog.gs_wlm_session_info_all S; + +CREATE OR REPLACE FUNCTION pg_catalog.create_wlm_session_info(IN flag int) +RETURNS int +AS $$ +DECLARE + query_str text; + record_cnt int; + BEGIN + record_cnt := 0; + + query_str := 'SELECT * FROM pg_catalog.pg_stat_get_wlm_session_info(1)'; + + IF flag > 0 THEN + EXECUTE 'INSERT INTO pg_catalog.gs_wlm_session_query_info_all ' || query_str; + ELSE + EXECUTE query_str; + END IF; + + RETURN record_cnt; + END; $$ +LANGUAGE plpgsql NOT FENCED; + +DROP FUNCTION IF EXISTS dbe_perf.global_slow_query_info() cascade; +DROP FUNCTION IF EXISTS dbe_perf.global_slow_query_history() cascade; +DROP VIEW IF EXISTS dbe_perf.gs_slow_query_info cascade; +DROP VIEW IF EXISTS dbe_perf.gs_slow_query_history cascade; + +CREATE OR REPLACE VIEW dbe_perf.gs_slow_query_info AS +SELECT + S.dbname, + S.schemaname, + S.nodename, + S.username, + S.queryid, + S.query, + S.start_time, + S.finish_time, + S.duration, + S.query_plan, + S.n_returned_rows, + S.n_tuples_fetched, + S.n_tuples_returned, + S.n_tuples_inserted, + S.n_tuples_updated, + S.n_tuples_deleted, + S.n_blocks_fetched, + S.n_blocks_hit, + S.db_time, + S.cpu_time, + S.execution_time, + S.parse_time, + S.plan_time, + S.rewrite_time, + S.pl_execution_time, + S.pl_compilation_time, + S.net_send_time, + S.data_io_time +FROM gs_wlm_session_query_info_all S where S.is_slow_query = 1; + +CREATE OR REPLACE VIEW dbe_perf.gs_slow_query_history AS +SELECT + S.dbname, + S.schemaname, + S.nodename, + S.username, + S.queryid, + S.query, + S.start_time, + S.finish_time, + S.duration, + S.query_plan, + S.n_returned_rows, + S.n_tuples_fetched, + S.n_tuples_returned, + S.n_tuples_inserted, + S.n_tuples_updated, + S.n_tuples_deleted, + S.n_blocks_fetched, + S.n_blocks_hit, + S.db_time, + S.cpu_time, + S.execution_time, + S.parse_time, + S.plan_time, + S.rewrite_time, + S.pl_execution_time, + S.pl_compilation_time, + S.net_send_time, + S.data_io_time +FROM pg_catalog.pg_stat_get_wlm_session_info(0) S where S.is_slow_query = 1; + +CREATE OR REPLACE FUNCTION dbe_perf.global_slow_query_history +RETURNS setof dbe_perf.gs_slow_query_history +AS $$ +DECLARE + row_data dbe_perf.gs_slow_query_history%rowtype; + row_name record; + query_str text; + query_str_nodes text; + BEGIN + --Get all the node names + query_str_nodes := 'SELECT node_name FROM pgxc_node WHERE node_type=''C'' AND nodeis_active = true'; + FOR row_name IN EXECUTE(query_str_nodes) LOOP + query_str := 'EXECUTE DIRECT ON (' || row_name.node_name || ') ''SELECT * FROM dbe_perf.gs_slow_query_history'''; + FOR row_data IN EXECUTE(query_str) LOOP + return next row_data; + END LOOP; + END LOOP; + return; + END; $$ +LANGUAGE 'plpgsql' NOT FENCED; + +CREATE OR REPLACE FUNCTION dbe_perf.global_slow_query_info() +RETURNS setof dbe_perf.gs_slow_query_info +AS $$ +DECLARE + row_data dbe_perf.gs_slow_query_info%rowtype; + row_name record; + query_str text; + query_str_nodes text; + BEGIN + --Get all the node names + query_str_nodes := 'SELECT node_name FROM pgxc_node WHERE node_type=''C'' AND nodeis_active = true'; + FOR row_name IN EXECUTE(query_str_nodes) LOOP + query_str := 'EXECUTE DIRECT ON (' || row_name.node_name || ') ''SELECT * FROM dbe_perf.gs_slow_query_info'''; + FOR row_data IN EXECUTE(query_str) LOOP + return next row_data; + END LOOP; + END LOOP; + return; + END; $$ +LANGUAGE 'plpgsql' NOT FENCED; + +CREATE OR REPLACE VIEW DBE_PERF.global_slow_query_history AS +SELECT * FROM DBE_PERF.global_slow_query_history(); + +CREATE OR REPLACE VIEW DBE_PERF.global_slow_query_info AS +SELECT * FROM DBE_PERF.global_slow_query_info(); + +GRANT SELECT ON TABLE pg_catalog.gs_wlm_session_query_info_all TO PUBLIC; +GRANT SELECT ON TABLE pg_catalog.gs_wlm_session_history TO PUBLIC; +GRANT SELECT ON TABLE pg_catalog.gs_wlm_session_info TO PUBLIC; +GRANT SELECT ON TABLE pg_catalog.gs_wlm_session_info_all TO PUBLIC; +GRANT SELECT ON TABLE DBE_PERF.STATEMENT TO PUBLIC; +GRANT SELECT ON TABLE DBE_PERF.summary_statement TO PUBLIC; +GRANT SELECT ON TABLE DBE_PERF.statement_history TO PUBLIC; +GRANT SELECT ON TABLE DBE_PERF.global_slow_query_history TO PUBLIC; +GRANT SELECT ON TABLE DBE_PERF.global_slow_query_info TO PUBLIC; + diff --git a/src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback_catalog_otherdb_92_906.sql b/src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback_catalog_otherdb_92_906.sql new file mode 100644 index 000000000..5b86b48b9 --- /dev/null +++ b/src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback_catalog_otherdb_92_906.sql @@ -0,0 +1,22 @@ +-- this table will be remove column +-- openGauss=# select table_schema, table_name,column_name from information_schema.columns where column_name = 'db_time'; +-- table_schema | table_name | column_name +-- --------------+-------------------------------+------------- +-- pg_catalog | gs_wlm_session_query_info_all | db_time --table +-- pg_catalog | statement_history | db_time --unlogged table, update in post-upgrade + +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all drop column if exists srt1_q cascade ; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all drop column if exists srt2_simple_query cascade ; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all drop column if exists srt3_analyze_rewrite cascade ; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all drop column if exists srt4_plan_query cascade ; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all drop column if exists srt5_light_query cascade ; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all drop column if exists srt6_p cascade ; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all drop column if exists srt7_b cascade ; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all drop column if exists srt8_e cascade ; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all drop column if exists srt9_d cascade ; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all drop column if exists srt10_s cascade ; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all drop column if exists srt11_c cascade ; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all drop column if exists srt12_u cascade ; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all drop column if exists srt13_before_query cascade ; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all drop column if exists srt14_after_query cascade ; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all drop column if exists rtt_unknown cascade ; diff --git a/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_906.sql b/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_906.sql index a720071e5..518497b40 100644 --- a/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_906.sql +++ b/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_906.sql @@ -13,3 +13,1130 @@ CREATE FUNCTION pg_catalog.ss_txnstatus_cache_stat( RETURNS SETOF record LANGUAGE internal STABLE NOT FENCED NOT SHIPPABLE ROWS 100 AS 'ss_txnstatus_cache_stat'; + +-- those type of view will recreate. +-- openGauss=# select table_schema, table_name,column_name from information_schema.columns where column_name = 'db_time'; +-- table_schema | table_name | column_name +-- --------------+-------------------------------+------------- +-- pg_catalog | gs_wlm_session_query_info_all | db_time -- table +-- pg_catalog | gs_wlm_session_info_all | db_time -- view +-- dbe_perf | statement | db_time -- view +-- pg_catalog | statement_history | db_time -- unlogged table +-- dbe_perf | statement_history | db_time -- view +-- dbe_perf | summary_statement | db_time -- view +-- dbe_perf | gs_slow_query_info | db_time -- view +-- dbe_perf | gs_slow_query_history | db_time -- view +-- dbe_perf | global_slow_query_history | db_time -- view +-- dbe_perf | global_slow_query_info | db_time -- view +-- those proc will recreated. +-- openGauss=# select proname, pronamespace from pg_proc where proargnames @> array['db_time']; +-- proname | pronamespace +-- ----------------------------------+-------------- +-- get_instr_unique_sql | 11 +-- pg_stat_get_wlm_session_info | 11 +-- standby_statement_history | 4988 +-- standby_statement_history | 4988 +-- get_global_full_sql_by_timestamp | 4988 +-- get_global_slow_sql_by_timestamp | 4988 + +/* We process history related tables、views and function now. */ +DROP VIEW IF EXISTS DBE_PERF.statement CASCADE; +DROP VIEW IF EXISTS DBE_PERF.summary_statement CASCADE; +DROP VIEW IF EXISTS DBE_PERF.statement_history CASCADE; +DROP FUNCTION IF EXISTS pg_catalog.get_instr_unique_sql() CASCADE; +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 5702; +CREATE FUNCTION pg_catalog.get_instr_unique_sql +( + OUT node_name name, + OUT node_id integer, + OUT user_name name, + OUT user_id oid, + OUT unique_sql_id bigint, + OUT query text, + OUT n_calls bigint, + OUT min_elapse_time bigint, + OUT max_elapse_time bigint, + OUT total_elapse_time bigint, + OUT n_returned_rows bigint, + OUT n_tuples_fetched bigint, + OUT n_tuples_returned bigint, + OUT n_tuples_inserted bigint, + OUT n_tuples_updated bigint, + OUT n_tuples_deleted bigint, + OUT n_blocks_fetched bigint, + OUT n_blocks_hit bigint, + OUT n_soft_parse bigint, + OUT n_hard_parse bigint, + OUT db_time bigint, + OUT cpu_time bigint, + OUT execution_time bigint, + OUT parse_time bigint, + OUT plan_time bigint, + OUT rewrite_time bigint, + OUT pl_execution_time bigint, + OUT pl_compilation_time bigint, + OUT data_io_time bigint, + OUT net_send_info text, + Out net_recv_info text, + OUT net_stream_send_info text, + OUT net_stream_recv_info text, + OUT last_updated timestamp with time zone, + OUT sort_count bigint, + OUT sort_time bigint, + OUT sort_mem_used bigint, + OUT sort_spill_count bigint, + OUT sort_spill_size bigint, + OUT hash_count bigint, + OUT hash_time bigint, + OUT hash_mem_used bigint, + OUT hash_spill_count bigint, + OUT hash_spill_size bigint, + OUT net_send_time bigint, + OUT srt1_q bigint, + OUT srt2_simple_query bigint, + OUT srt3_analyze_rewrite bigint, + OUT srt4_plan_query bigint, + OUT srt5_light_query bigint, + OUT srt6_p bigint, + OUT srt7_b bigint, + OUT srt8_e bigint, + OUT srt9_d bigint, + OUT srt10_s bigint, + OUT srt11_c bigint, + OUT srt12_u bigint, + OUT srt13_before_query bigint, + OUT srt14_after_query bigint, + OUT rtt_unknown bigint +) +RETURNS setof record LANGUAGE INTERNAL VOLATILE NOT FENCED as 'get_instr_unique_sql'; + +DROP INDEX IF EXISTS pg_catalog.statement_history_time_idx; +DROP TABLE IF EXISTS pg_catalog.statement_history cascade; + +CREATE unlogged table IF NOT EXISTS pg_catalog.statement_history( + db_name name, + schema_name name, + origin_node integer, + user_name name, + application_name text, + client_addr text, + client_port integer, + unique_query_id bigint, + debug_query_id bigint, + query text, + start_time timestamp with time zone, + finish_time timestamp with time zone, + slow_sql_threshold bigint, + transaction_id bigint, + thread_id bigint, + session_id bigint, + n_soft_parse bigint, + n_hard_parse bigint, + query_plan text, + n_returned_rows bigint, + n_tuples_fetched bigint, + n_tuples_returned bigint, + n_tuples_inserted bigint, + n_tuples_updated bigint, + n_tuples_deleted bigint, + n_blocks_fetched bigint, + n_blocks_hit bigint, + db_time bigint, + cpu_time bigint, + execution_time bigint, + parse_time bigint, + plan_time bigint, + rewrite_time bigint, + pl_execution_time bigint, + pl_compilation_time bigint, + data_io_time bigint, + net_send_info text, + net_recv_info text, + net_stream_send_info text, + net_stream_recv_info text, + lock_count bigint, + lock_time bigint, + lock_wait_count bigint, + lock_wait_time bigint, + lock_max_count bigint, + lwlock_count bigint, + lwlock_wait_count bigint, + lwlock_time bigint, + lwlock_wait_time bigint, + details bytea, + is_slow_sql boolean, + trace_id text, + advise text, + net_send_time bigint, + srt1_q bigint, + srt2_simple_query bigint, + srt3_analyze_rewrite bigint, + srt4_plan_query bigint, + srt5_light_query bigint, + srt6_p bigint, + srt7_b bigint, + srt8_e bigint, + srt9_d bigint, + srt10_s bigint, + srt11_c bigint, + srt12_u bigint, + srt13_before_query bigint, + srt14_after_query bigint, + rtt_unknown bigint +); +REVOKE ALL on table pg_catalog.statement_history FROM public; +create index pg_catalog.statement_history_time_idx on pg_catalog.statement_history USING btree (start_time, is_slow_sql); + +DROP FUNCTION IF EXISTS dbe_perf.standby_statement_history(boolean); +DROP FUNCTION IF EXISTS dbe_perf.standby_statement_history(boolean, timestamp with time zone[]); +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 3118; +CREATE FUNCTION dbe_perf.standby_statement_history( + IN only_slow boolean, + OUT db_name name, + OUT schema_name name, + OUT origin_node integer, + OUT user_name name, + OUT application_name text, + OUT client_addr text, + OUT client_port integer, + OUT unique_query_id bigint, + OUT debug_query_id bigint, + OUT query text, + OUT start_time timestamp with time zone, + OUT finish_time timestamp with time zone, + OUT slow_sql_threshold bigint, + OUT transaction_id bigint, + OUT thread_id bigint, + OUT session_id bigint, + OUT n_soft_parse bigint, + OUT n_hard_parse bigint, + OUT query_plan text, + OUT n_returned_rows bigint, + OUT n_tuples_fetched bigint, + OUT n_tuples_returned bigint, + OUT n_tuples_inserted bigint, + OUT n_tuples_updated bigint, + OUT n_tuples_deleted bigint, + OUT n_blocks_fetched bigint, + OUT n_blocks_hit bigint, + OUT db_time bigint, + OUT cpu_time bigint, + OUT execution_time bigint, + OUT parse_time bigint, + OUT plan_time bigint, + OUT rewrite_time bigint, + OUT pl_execution_time bigint, + OUT pl_compilation_time bigint, + OUT data_io_time bigint, + OUT net_send_info text, + OUT net_recv_info text, + OUT net_stream_send_info text, + OUT net_stream_recv_info text, + OUT lock_count bigint, + OUT lock_time bigint, + OUT lock_wait_count bigint, + OUT lock_wait_time bigint, + OUT lock_max_count bigint, + OUT lwlock_count bigint, + OUT lwlock_wait_count bigint, + OUT lwlock_time bigint, + OUT lwlock_wait_time bigint, + OUT details bytea, + OUT is_slow_sql boolean, + OUT trace_id text, + OUT advise text, + OUT net_send_time bigint, + OUT srt1_q bigint, + OUT srt2_simple_query bigint, + OUT srt3_analyze_rewrite bigint, + OUT srt4_plan_query bigint, + OUT srt5_light_query bigint, + OUT srt6_p bigint, + OUT srt7_b bigint, + OUT srt8_e bigint, + OUT srt9_d bigint, + OUT srt10_s bigint, + OUT srt11_c bigint, + OUT srt12_u bigint, + OUT srt13_before_query bigint, + OUT srt14_after_query bigint, + OUT rtt_unknown bigint) +RETURNS SETOF record NOT FENCED NOT SHIPPABLE ROWS 10000 +LANGUAGE internal AS $function$standby_statement_history_1v$function$; + +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 3119; +CREATE FUNCTION dbe_perf.standby_statement_history( + IN only_slow boolean, + VARIADIC finish_time timestamp with time zone[], + OUT db_name name, + OUT schema_name name, + OUT origin_node integer, + OUT user_name name, + OUT application_name text, + OUT client_addr text, + OUT client_port integer, + OUT unique_query_id bigint, + OUT debug_query_id bigint, + OUT query text, + OUT start_time timestamp with time zone, + OUT finish_time timestamp with time zone, + OUT slow_sql_threshold bigint, + OUT transaction_id bigint, + OUT thread_id bigint, + OUT session_id bigint, + OUT n_soft_parse bigint, + OUT n_hard_parse bigint, + OUT query_plan text, + OUT n_returned_rows bigint, + OUT n_tuples_fetched bigint, + OUT n_tuples_returned bigint, + OUT n_tuples_inserted bigint, + OUT n_tuples_updated bigint, + OUT n_tuples_deleted bigint, + OUT n_blocks_fetched bigint, + OUT n_blocks_hit bigint, + OUT db_time bigint, + OUT cpu_time bigint, + OUT execution_time bigint, + OUT parse_time bigint, + OUT plan_time bigint, + OUT rewrite_time bigint, + OUT pl_execution_time bigint, + OUT pl_compilation_time bigint, + OUT data_io_time bigint, + OUT net_send_info text, + OUT net_recv_info text, + OUT net_stream_send_info text, + OUT net_stream_recv_info text, + OUT lock_count bigint, + OUT lock_time bigint, + OUT lock_wait_count bigint, + OUT lock_wait_time bigint, + OUT lock_max_count bigint, + OUT lwlock_count bigint, + OUT lwlock_wait_count bigint, + OUT lwlock_time bigint, + OUT lwlock_wait_time bigint, + OUT details bytea, + OUT is_slow_sql boolean, + OUT net_send_time bigint, + OUT srt1_q bigint, + OUT srt2_simple_query bigint, + OUT srt3_analyze_rewrite bigint, + OUT srt4_plan_query bigint, + OUT srt5_light_query bigint, + OUT srt6_p bigint, + OUT srt7_b bigint, + OUT srt8_e bigint, + OUT srt9_d bigint, + OUT srt10_s bigint, + OUT srt11_c bigint, + OUT srt12_u bigint, + OUT srt13_before_query bigint, + OUT srt14_after_query bigint, + OUT rtt_unknown bigint) +RETURNS SETOF record NOT FENCED NOT SHIPPABLE ROWS 10000 +LANGUAGE internal AS $function$standby_statement_history$function$; + +CREATE VIEW DBE_PERF.statement AS + SELECT * FROM get_instr_unique_sql(); + +DROP FUNCTION IF EXISTS dbe_perf.get_summary_statement() cascade; +CREATE OR REPLACE FUNCTION dbe_perf.get_summary_statement() +RETURNS setof dbe_perf.statement +AS $$ +DECLARE + row_data dbe_perf.statement%rowtype; + row_name record; + query_str text; + query_str_nodes text; + BEGIN + --Get all the node names + query_str_nodes := 'select * from dbe_perf.node_name'; + FOR row_name IN EXECUTE(query_str_nodes) LOOP + query_str := 'SELECT * FROM dbe_perf.statement'; + FOR row_data IN EXECUTE(query_str) LOOP + return next row_data; + END LOOP; + END LOOP; + return; + END; $$ +LANGUAGE 'plpgsql' NOT FENCED; + +CREATE VIEW DBE_PERF.summary_statement AS + SELECT * FROM DBE_PERF.get_summary_statement(); + +CREATE VIEW DBE_PERF.statement_history AS + select * from pg_catalog.statement_history; + +DROP FUNCTION IF EXISTS dbe_perf.get_global_full_sql_by_timestamp(timestamp with time zone, timestamp with time zone); +CREATE OR REPLACE FUNCTION dbe_perf.get_global_full_sql_by_timestamp + (in start_timestamp timestamp with time zone, + in end_timestamp timestamp with time zone, + OUT node_name name, + OUT db_name name, + OUT schema_name name, + OUT origin_node integer, + OUT user_name name, + OUT application_name text, + OUT client_addr text, + OUT client_port integer, + OUT unique_query_id bigint, + OUT debug_query_id bigint, + OUT query text, + OUT start_time timestamp with time zone, + OUT finish_time timestamp with time zone, + OUT slow_sql_threshold bigint, + OUT transaction_id bigint, + OUT thread_id bigint, + OUT session_id bigint, + OUT n_soft_parse bigint, + OUT n_hard_parse bigint, + OUT query_plan text, + OUT n_returned_rows bigint, + OUT n_tuples_fetched bigint, + OUT n_tuples_returned bigint, + OUT n_tuples_inserted bigint, + OUT n_tuples_updated bigint, + OUT n_tuples_deleted bigint, + OUT n_blocks_fetched bigint, + OUT n_blocks_hit bigint, + OUT db_time bigint, + OUT cpu_time bigint, + OUT execution_time bigint, + OUT parse_time bigint, + OUT plan_time bigint, + OUT rewrite_time bigint, + OUT pl_execution_time bigint, + OUT pl_compilation_time bigint, + OUT data_io_time bigint, + OUT net_send_info text, + OUT net_recv_info text, + OUT net_stream_send_info text, + OUT net_stream_recv_info text, + OUT lock_count bigint, + OUT lock_time bigint, + OUT lock_wait_count bigint, + OUT lock_wait_time bigint, + OUT lock_max_count bigint, + OUT lwlock_count bigint, + OUT lwlock_wait_count bigint, + OUT lwlock_time bigint, + OUT lwlock_wait_time bigint, + OUT details bytea, + OUT is_slow_sql bool, + OUT trace_id text, + OUT advise text, + OUT net_send_time bigint, + OUT srt1_q bigint, + OUT srt2_simple_query bigint, + OUT srt3_analyze_rewrite bigint, + OUT srt4_plan_query bigint, + OUT srt5_light_query bigint, + OUT srt6_p bigint, + OUT srt7_b bigint, + OUT srt8_e bigint, + OUT srt9_d bigint, + OUT srt10_s bigint, + OUT srt11_c bigint, + OUT srt12_u bigint, + OUT srt13_before_query bigint, + OUT srt14_after_query bigint, + OUT rtt_unknown bigint + ) + RETURNS setof record + AS $$ + DECLARE + row_data pg_catalog.statement_history%rowtype; + row_name record; + query_str text; + -- node name + query_str_nodes text; + BEGIN + -- Get all node names(CN + master DN) + query_str_nodes := 'select * from dbe_perf.node_name'; + FOR row_name IN EXECUTE(query_str_nodes) LOOP + query_str := 'SELECT * FROM DBE_PERF.statement_history where start_time >= ''' ||$1|| ''' and start_time <= ''' || $2 || ''''; + FOR row_data IN EXECUTE(query_str) LOOP + node_name := row_name.node_name; + db_name := row_data.db_name; + schema_name := row_data.schema_name; + origin_node := row_data.origin_node; + user_name := row_data.user_name; + application_name := row_data.application_name; + client_addr := row_data.client_addr; + client_port := row_data.client_port; + unique_query_id := row_data.unique_query_id; + debug_query_id := row_data.debug_query_id; + query := row_data.query; + start_time := row_data.start_time; + finish_time := row_data.finish_time; + slow_sql_threshold := row_data.slow_sql_threshold; + transaction_id := row_data.transaction_id; + thread_id := row_data.thread_id; + session_id := row_data.session_id; + n_soft_parse := row_data.n_soft_parse; + n_hard_parse := row_data.n_hard_parse; + query_plan := row_data.query_plan; + n_returned_rows := row_data.n_returned_rows; + n_tuples_fetched := row_data.n_tuples_fetched; + n_tuples_returned := row_data.n_tuples_returned; + n_tuples_inserted := row_data.n_tuples_inserted; + n_tuples_updated := row_data.n_tuples_updated; + n_tuples_deleted := row_data.n_tuples_deleted; + n_blocks_fetched := row_data.n_blocks_fetched; + n_blocks_hit := row_data.n_blocks_hit; + db_time := row_data.db_time; + cpu_time := row_data.cpu_time; + execution_time := row_data.execution_time; + parse_time := row_data.parse_time; + plan_time := row_data.plan_time; + rewrite_time := row_data.rewrite_time; + pl_execution_time := row_data.pl_execution_time; + pl_compilation_time := row_data.pl_compilation_time; + data_io_time := row_data.data_io_time; + net_send_info := row_data.net_send_info; + net_recv_info := row_data.net_recv_info; + net_stream_send_info := row_data.net_stream_send_info; + net_stream_recv_info := row_data.net_stream_recv_info; + lock_count := row_data.lock_count; + lock_time := row_data.lock_time; + lock_wait_count := row_data.lock_wait_count; + lock_wait_time := row_data.lock_wait_time; + lock_max_count := row_data.lock_max_count; + lwlock_count := row_data.lwlock_count; + lwlock_wait_count := row_data.lwlock_wait_count; + lwlock_time := row_data.lwlock_time; + lwlock_wait_time := row_data.lwlock_wait_time; + details := row_data.details; + is_slow_sql := row_data.is_slow_sql; + trace_id := row_data.trace_id; + advise := row_data.advise; + net_send_time =row_data.net_send_time; + srt1_q := row_data.srt1_q; + srt2_simple_query := row_data.srt2_simple_query; + srt3_analyze_rewrite := row_data.srt3_analyze_rewrite; + srt4_plan_query := row_data.srt4_plan_query; + srt5_light_query := row_data.srt5_light_query; + srt6_p := row_data.srt6_p; + srt7_b := row_data.srt7_b; + srt8_e := row_data.srt8_e; + srt9_d := row_data.srt9_d; + srt10_s := row_data.srt10_s; + srt11_c := row_data.srt11_c; + srt12_u := row_data.srt12_u; + srt13_before_query := row_data.srt13_before_query; + srt14_after_query := row_data.srt14_after_query; + rtt_unknown := row_data.rtt_unknown; + return next; + END LOOP; + END LOOP; + return; + END; $$ +LANGUAGE 'plpgsql' NOT FENCED; + +DROP FUNCTION IF EXISTS dbe_perf.get_global_slow_sql_by_timestamp(timestamp with time zone, timestamp with time zone); +CREATE OR REPLACE FUNCTION DBE_PERF.get_global_slow_sql_by_timestamp + (in start_timestamp timestamp with time zone, + in end_timestamp timestamp with time zone, + OUT node_name name, + OUT db_name name, + OUT schema_name name, + OUT origin_node integer, + OUT user_name name, + OUT application_name text, + OUT client_addr text, + OUT client_port integer, + OUT unique_query_id bigint, + OUT debug_query_id bigint, + OUT query text, + OUT start_time timestamp with time zone, + OUT finish_time timestamp with time zone, + OUT slow_sql_threshold bigint, + OUT transaction_id bigint, + OUT thread_id bigint, + OUT session_id bigint, + OUT n_soft_parse bigint, + OUT n_hard_parse bigint, + OUT query_plan text, + OUT n_returned_rows bigint, + OUT n_tuples_fetched bigint, + OUT n_tuples_returned bigint, + OUT n_tuples_inserted bigint, + OUT n_tuples_updated bigint, + OUT n_tuples_deleted bigint, + OUT n_blocks_fetched bigint, + OUT n_blocks_hit bigint, + OUT db_time bigint, + OUT cpu_time bigint, + OUT execution_time bigint, + OUT parse_time bigint, + OUT plan_time bigint, + OUT rewrite_time bigint, + OUT pl_execution_time bigint, + OUT pl_compilation_time bigint, + OUT data_io_time bigint, + OUT net_send_info text, + OUT net_recv_info text, + OUT net_stream_send_info text, + OUT net_stream_recv_info text, + OUT lock_count bigint, + OUT lock_time bigint, + OUT lock_wait_count bigint, + OUT lock_wait_time bigint, + OUT lock_max_count bigint, + OUT lwlock_count bigint, + OUT lwlock_wait_count bigint, + OUT lwlock_time bigint, + OUT lwlock_wait_time bigint, + OUT details bytea, + OUT is_slow_sql bool, + OUT trace_id text, + OUT advise text, + OUT net_send_time bigint, + OUT srt1_q bigint, + OUT srt2_simple_query bigint, + OUT srt3_analyze_rewrite bigint, + OUT srt4_plan_query bigint, + OUT srt5_light_query bigint, + OUT srt6_p bigint, + OUT srt7_b bigint, + OUT srt8_e bigint, + OUT srt9_d bigint, + OUT srt10_s bigint, + OUT srt11_c bigint, + OUT srt12_u bigint, + OUT srt13_before_query bigint, + OUT srt14_after_query bigint, + OUT rtt_unknown bigint) + RETURNS setof record + AS $$ + DECLARE + row_data pg_catalog.statement_history%rowtype; + row_name record; + query_str text; + -- node name + query_str_nodes text; + BEGIN + -- Get all node names(CN + master DN) + query_str_nodes := 'select * from dbe_perf.node_name'; + FOR row_name IN EXECUTE(query_str_nodes) LOOP + query_str := 'SELECT * FROM DBE_PERF.statement_history where start_time >= ''' ||$1|| ''' and start_time <= ''' || $2 || ''' and is_slow_sql = true '; + FOR row_data IN EXECUTE(query_str) LOOP + node_name := row_name.node_name; + db_name := row_data.db_name; + schema_name := row_data.schema_name; + origin_node := row_data.origin_node; + user_name := row_data.user_name; + application_name := row_data.application_name; + client_addr := row_data.client_addr; + client_port := row_data.client_port; + unique_query_id := row_data.unique_query_id; + debug_query_id := row_data.debug_query_id; + query := row_data.query; + start_time := row_data.start_time; + finish_time := row_data.finish_time; + slow_sql_threshold := row_data.slow_sql_threshold; + transaction_id := row_data.transaction_id; + thread_id := row_data.thread_id; + session_id := row_data.session_id; + n_soft_parse := row_data.n_soft_parse; + n_hard_parse := row_data.n_hard_parse; + query_plan := row_data.query_plan; + n_returned_rows := row_data.n_returned_rows; + n_tuples_fetched := row_data.n_tuples_fetched; + n_tuples_returned := row_data.n_tuples_returned; + n_tuples_inserted := row_data.n_tuples_inserted; + n_tuples_updated := row_data.n_tuples_updated; + n_tuples_deleted := row_data.n_tuples_deleted; + n_blocks_fetched := row_data.n_blocks_fetched; + n_blocks_hit := row_data.n_blocks_hit; + db_time := row_data.db_time; + cpu_time := row_data.cpu_time; + execution_time := row_data.execution_time; + parse_time := row_data.parse_time; + plan_time := row_data.plan_time; + rewrite_time := row_data.rewrite_time; + pl_execution_time := row_data.pl_execution_time; + pl_compilation_time := row_data.pl_compilation_time; + data_io_time := row_data.data_io_time; + net_send_info := row_data.net_send_info; + net_recv_info := row_data.net_recv_info; + net_stream_send_info := row_data.net_stream_send_info; + net_stream_recv_info := row_data.net_stream_recv_info; + lock_count := row_data.lock_count; + lock_time := row_data.lock_time; + lock_wait_count := row_data.lock_wait_count; + lock_wait_time := row_data.lock_wait_time; + lock_max_count := row_data.lock_max_count; + lwlock_count := row_data.lwlock_count; + lwlock_wait_count := row_data.lwlock_wait_count; + lwlock_time := row_data.lwlock_time; + lwlock_wait_time := row_data.lwlock_wait_time; + details := row_data.details; + is_slow_sql := row_data.is_slow_sql; + trace_id := row_data.trace_id; + advise := row_data.advise; + net_send_time =row_data.net_send_time; + srt1_q := row_data.srt1_q; + srt2_simple_query := row_data.srt2_simple_query; + srt3_analyze_rewrite := row_data.srt3_analyze_rewrite; + srt4_plan_query := row_data.srt4_plan_query; + srt5_light_query := row_data.srt5_light_query; + srt6_p := row_data.srt6_p; + srt7_b := row_data.srt7_b; + srt8_e := row_data.srt8_e; + srt9_d := row_data.srt9_d; + srt10_s := row_data.srt10_s; + srt11_c := row_data.srt11_c; + srt12_u := row_data.srt12_u; + srt13_before_query := row_data.srt13_before_query; + srt14_after_query := row_data.srt14_after_query; + rtt_unknown := row_data.rtt_unknown; + return next; + END LOOP; + END LOOP; + return; + END; $$ +LANGUAGE 'plpgsql' NOT FENCED; + +/* we process wlm releates.*/ +DROP VIEW IF EXISTS pg_catalog.gs_wlm_session_info_all CASCADE; +DROP VIEW IF EXISTS pg_catalog.gs_wlm_session_info CASCADE; +DROP VIEW IF EXISTS pg_catalog.gs_wlm_session_info CASCADE; +DROP FUNCTION IF EXISTS pg_catalog.create_wlm_session_info(IN flag int) CASCADE; +DROP FUNCTION IF EXISTS pg_catalog.pg_stat_get_wlm_session_info(OID) cascade; + +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 5002; +CREATE OR REPLACE FUNCTION pg_catalog.pg_stat_get_wlm_session_info +(OID, + OUT datid oid, + OUT dbname text, + OUT schemaname text, + OUT nodename text, + OUT username text, + OUT application_name text, + OUT client_addr inet, + OUT client_hostname text, + OUT client_port integer, + OUT query_band text, + OUT block_time bigint, + OUT start_time timestamp with time zone, + OUT finish_time timestamp with time zone, + OUT duration bigint, + OUT estimate_total_time bigint, + OUT status text, + OUT abort_info text, + OUT resource_pool text, + OUT control_group text, + OUT estimate_memory integer, + OUT min_peak_memory integer, + OUT max_peak_memory integer, + OUT average_peak_memory integer, + OUT memory_skew_percent integer, + OUT spill_info text, + OUT min_spill_size integer, + OUT max_spill_size integer, + OUT average_spill_size integer, + OUT spill_skew_percent integer, + OUT min_dn_time bigint, + OUT max_dn_time bigint, + OUT average_dn_time bigint, + OUT dntime_skew_percent integer, + OUT min_cpu_time bigint, + OUT max_cpu_time bigint, + OUT total_cpu_time bigint, + OUT cpu_skew_percent integer, + OUT min_peak_iops integer, + OUT max_peak_iops integer, + OUT average_peak_iops integer, + OUT iops_skew_percent integer, + OUT warning text, + OUT queryid bigint, + OUT query text, + OUT query_plan text, + OUT node_group text, + OUT cpu_top1_node_name text, + OUT cpu_top2_node_name text, + OUT cpu_top3_node_name text, + OUT cpu_top4_node_name text, + OUT cpu_top5_node_name text, + OUT mem_top1_node_name text, + OUT mem_top2_node_name text, + OUT mem_top3_node_name text, + OUT mem_top4_node_name text, + OUT mem_top5_node_name text, + OUT cpu_top1_value bigint, + OUT cpu_top2_value bigint, + OUT cpu_top3_value bigint, + OUT cpu_top4_value bigint, + OUT cpu_top5_value bigint, + OUT mem_top1_value bigint, + OUT mem_top2_value bigint, + OUT mem_top3_value bigint, + OUT mem_top4_value bigint, + OUT mem_top5_value bigint, + OUT top_mem_dn text, + OUT top_cpu_dn text, + OUT n_returned_rows bigint, + OUT n_tuples_fetched bigint, + OUT n_tuples_returned bigint, + OUT n_tuples_inserted bigint, + OUT n_tuples_updated bigint, + OUT n_tuples_deleted bigint, + OUT n_blocks_fetched bigint, + OUT n_blocks_hit bigint, + OUT db_time bigint, + OUT cpu_time bigint, + OUT execution_time bigint, + OUT parse_time bigint, + OUT plan_time bigint, + OUT rewrite_time bigint, + OUT pl_execution_time bigint, + OUT pl_compilation_time bigint, + OUT net_send_time bigint, + OUT data_io_time bigint, + OUT is_slow_query bigint, + OUT srt1_q bigint, + OUT srt2_simple_query bigint, + OUT srt3_analyze_rewrite bigint, + OUT srt4_plan_query bigint, + OUT srt5_light_query bigint, + OUT srt6_p bigint, + OUT srt7_b bigint, + OUT srt8_e bigint, + OUT srt9_d bigint, + OUT srt10_s bigint, + OUT srt11_c bigint, + OUT srt12_u bigint, + OUT srt13_before_query bigint, + OUT srt14_after_query bigint, + OUT rtt_unknown bigint + ) RETURNS setof record LANGUAGE INTERNAL VOLATILE NOT FENCED as 'pg_stat_get_wlm_session_info'; + + +CREATE VIEW pg_catalog.gs_wlm_session_info_all AS +SELECT * FROM pg_catalog.pg_stat_get_wlm_session_info(0); + +--process wlm_session info +CREATE VIEW pg_catalog.gs_wlm_session_info AS +SELECT + S.datid, + S.dbname, + S.schemaname, + S.nodename, + S.username, + S.application_name, + S.client_addr, + S.client_hostname, + S.client_port, + S.query_band, + S.block_time, + S.start_time, + S.finish_time, + S.duration, + S.estimate_total_time, + S.status, + S.abort_info, + S.resource_pool, + S.control_group, + S.estimate_memory, + S.min_peak_memory, + S.max_peak_memory, + S.average_peak_memory, + S.memory_skew_percent, + S.spill_info, + S.min_spill_size, + S.max_spill_size, + S.average_spill_size, + S.spill_skew_percent, + S.min_dn_time, + S.max_dn_time, + S.average_dn_time, + S.dntime_skew_percent, + S.min_cpu_time, + S.max_cpu_time, + S.total_cpu_time, + S.cpu_skew_percent, + S.min_peak_iops, + S.max_peak_iops, + S.average_peak_iops, + S.iops_skew_percent, + S.warning, + S.queryid, + S.query, + S.query_plan, + S.node_group, + S.cpu_top1_node_name, + S.cpu_top2_node_name, + S.cpu_top3_node_name, + S.cpu_top4_node_name, + S.cpu_top5_node_name, + S.mem_top1_node_name, + S.mem_top2_node_name, + S.mem_top3_node_name, + S.mem_top4_node_name, + S.mem_top5_node_name, + S.cpu_top1_value, + S.cpu_top2_value, + S.cpu_top3_value, + S.cpu_top4_value, + S.cpu_top5_value, + S.mem_top1_value, + S.mem_top2_value, + S.mem_top3_value, + S.mem_top4_value, + S.mem_top5_value, + S.top_mem_dn, + S.top_cpu_dn +FROM gs_wlm_session_query_info_all S; + +CREATE VIEW pg_catalog.gs_wlm_session_history AS +SELECT + S.datid, + S.dbname, + S.schemaname, + S.nodename, + S.username, + S.application_name, + S.client_addr, + S.client_hostname, + S.client_port, + S.query_band, + S.block_time, + S.start_time, + S.finish_time, + S.duration, + S.estimate_total_time, + S.status, + S.abort_info, + S.resource_pool, + S.control_group, + S.estimate_memory, + S.min_peak_memory, + S.max_peak_memory, + S.average_peak_memory, + S.memory_skew_percent, + S.spill_info, + S.min_spill_size, + S.max_spill_size, + S.average_spill_size, + S.spill_skew_percent, + S.min_dn_time, + S.max_dn_time, + S.average_dn_time, + S.dntime_skew_percent, + S.min_cpu_time, + S.max_cpu_time, + S.total_cpu_time, + S.cpu_skew_percent, + S.min_peak_iops, + S.max_peak_iops, + S.average_peak_iops, + S.iops_skew_percent, + S.warning, + S.queryid, + S.query, + S.query_plan, + S.node_group, + S.cpu_top1_node_name, + S.cpu_top2_node_name, + S.cpu_top3_node_name, + S.cpu_top4_node_name, + S.cpu_top5_node_name, + S.mem_top1_node_name, + S.mem_top2_node_name, + S.mem_top3_node_name, + S.mem_top4_node_name, + S.mem_top5_node_name, + S.cpu_top1_value, + S.cpu_top2_value, + S.cpu_top3_value, + S.cpu_top4_value, + S.cpu_top5_value, + S.mem_top1_value, + S.mem_top2_value, + S.mem_top3_value, + S.mem_top4_value, + S.mem_top5_value, + S.top_mem_dn, + S.top_cpu_dn +FROM pg_catalog.gs_wlm_session_info_all S; + +CREATE OR REPLACE FUNCTION pg_catalog.create_wlm_session_info(IN flag int) +RETURNS int +AS $$ +DECLARE + query_str text; + record_cnt int; + BEGIN + record_cnt := 0; + + query_str := 'SELECT * FROM pg_catalog.pg_stat_get_wlm_session_info(1)'; + + IF flag > 0 THEN + EXECUTE 'INSERT INTO pg_catalog.gs_wlm_session_query_info_all ' || query_str; + ELSE + EXECUTE query_str; + END IF; + + RETURN record_cnt; + END; $$ +LANGUAGE plpgsql NOT FENCED; + +DROP FUNCTION IF EXISTS dbe_perf.global_slow_query_info() cascade; +DROP FUNCTION IF EXISTS dbe_perf.global_slow_query_history() cascade; +DROP VIEW IF EXISTS dbe_perf.gs_slow_query_info cascade; +DROP VIEW IF EXISTS dbe_perf.gs_slow_query_history cascade; + +CREATE OR REPLACE VIEW dbe_perf.gs_slow_query_info AS +SELECT + S.dbname, + S.schemaname, + S.nodename, + S.username, + S.queryid, + S.query, + S.start_time, + S.finish_time, + S.duration, + S.query_plan, + S.n_returned_rows, + S.n_tuples_fetched, + S.n_tuples_returned, + S.n_tuples_inserted, + S.n_tuples_updated, + S.n_tuples_deleted, + S.n_blocks_fetched, + S.n_blocks_hit, + S.db_time, + S.cpu_time, + S.execution_time, + S.parse_time, + S.plan_time, + S.rewrite_time, + S.pl_execution_time, + S.pl_compilation_time, + S.net_send_time, + S.data_io_time, + S.srt1_q, + S.srt2_simple_query, + S.srt3_analyze_rewrite, + S.srt4_plan_query, + S.srt5_light_query, + S.srt6_p, + S.srt7_b, + S.srt8_e, + S.srt9_d, + S.srt10_s, + S.srt11_c, + S.srt12_u, + S.srt13_before_query, + S.srt14_after_query, + S.rtt_unknown +FROM gs_wlm_session_query_info_all S where S.is_slow_query = 1; + +CREATE OR REPLACE VIEW dbe_perf.gs_slow_query_history AS +SELECT + S.dbname, + S.schemaname, + S.nodename, + S.username, + S.queryid, + S.query, + S.start_time, + S.finish_time, + S.duration, + S.query_plan, + S.n_returned_rows, + S.n_tuples_fetched, + S.n_tuples_returned, + S.n_tuples_inserted, + S.n_tuples_updated, + S.n_tuples_deleted, + S.n_blocks_fetched, + S.n_blocks_hit, + S.db_time, + S.cpu_time, + S.execution_time, + S.parse_time, + S.plan_time, + S.rewrite_time, + S.pl_execution_time, + S.pl_compilation_time, + S.net_send_time, + S.data_io_time, + S.srt1_q, + S.srt2_simple_query, + S.srt3_analyze_rewrite, + S.srt4_plan_query, + S.srt5_light_query, + S.srt6_p, + S.srt7_b, + S.srt8_e, + S.srt9_d, + S.srt10_s, + S.srt11_c, + S.srt12_u, + S.srt13_before_query, + S.srt14_after_query, + S.rtt_unknown +FROM pg_catalog.pg_stat_get_wlm_session_info(0) S where S.is_slow_query = 1; + +CREATE OR REPLACE FUNCTION dbe_perf.global_slow_query_history +RETURNS setof dbe_perf.gs_slow_query_history +AS $$ +DECLARE + row_data dbe_perf.gs_slow_query_history%rowtype; + row_name record; + query_str text; + query_str_nodes text; + BEGIN + --Get all the node names + query_str_nodes := 'SELECT node_name FROM pgxc_node WHERE node_type=''C'' AND nodeis_active = true'; + FOR row_name IN EXECUTE(query_str_nodes) LOOP + query_str := 'EXECUTE DIRECT ON (' || row_name.node_name || ') ''SELECT * FROM dbe_perf.gs_slow_query_history'''; + FOR row_data IN EXECUTE(query_str) LOOP + return next row_data; + END LOOP; + END LOOP; + return; + END; $$ +LANGUAGE 'plpgsql' NOT FENCED; + +CREATE OR REPLACE FUNCTION dbe_perf.global_slow_query_info() +RETURNS setof dbe_perf.gs_slow_query_info +AS $$ +DECLARE + row_data dbe_perf.gs_slow_query_info%rowtype; + row_name record; + query_str text; + query_str_nodes text; + BEGIN + --Get all the node names + query_str_nodes := 'SELECT node_name FROM pgxc_node WHERE node_type=''C'' AND nodeis_active = true'; + FOR row_name IN EXECUTE(query_str_nodes) LOOP + query_str := 'EXECUTE DIRECT ON (' || row_name.node_name || ') ''SELECT * FROM dbe_perf.gs_slow_query_info'''; + FOR row_data IN EXECUTE(query_str) LOOP + return next row_data; + END LOOP; + END LOOP; + return; + END; $$ +LANGUAGE 'plpgsql' NOT FENCED; + +CREATE OR REPLACE VIEW DBE_PERF.global_slow_query_history AS +SELECT * FROM DBE_PERF.global_slow_query_history(); + +CREATE OR REPLACE VIEW DBE_PERF.global_slow_query_info AS +SELECT * FROM DBE_PERF.global_slow_query_info(); + +GRANT SELECT ON TABLE pg_catalog.gs_wlm_session_query_info_all TO PUBLIC; +GRANT SELECT ON TABLE pg_catalog.gs_wlm_session_history TO PUBLIC; +GRANT SELECT ON TABLE pg_catalog.gs_wlm_session_info TO PUBLIC; +GRANT SELECT ON TABLE pg_catalog.gs_wlm_session_info_all TO PUBLIC; +GRANT SELECT ON TABLE DBE_PERF.STATEMENT TO PUBLIC; +GRANT SELECT ON TABLE DBE_PERF.summary_statement TO PUBLIC; +GRANT SELECT ON TABLE DBE_PERF.statement_history TO PUBLIC; +GRANT SELECT ON TABLE DBE_PERF.global_slow_query_history TO PUBLIC; +GRANT SELECT ON TABLE DBE_PERF.global_slow_query_info TO PUBLIC; \ No newline at end of file diff --git a/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade_catalog_maindb_92_906.sql b/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade_catalog_maindb_92_906.sql new file mode 100644 index 000000000..51a105b0b --- /dev/null +++ b/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade_catalog_maindb_92_906.sql @@ -0,0 +1,22 @@ +-- this table will be add column +-- openGauss=# select table_schema, table_name,column_name from information_schema.columns where column_name = 'db_time'; +-- table_schema | table_name | column_name +-- --------------+-------------------------------+------------- +-- pg_catalog | gs_wlm_session_query_info_all | db_time --table +-- pg_catalog | statement_history | db_time --unlogged table, update in post-upgrade + +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all add column srt1_q bigint; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all add column srt2_simple_query bigint; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all add column srt3_analyze_rewrite bigint; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all add column srt4_plan_query bigint; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all add column srt5_light_query bigint; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all add column srt6_p bigint; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all add column srt7_b bigint; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all add column srt8_e bigint; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all add column srt9_d bigint; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all add column srt10_s bigint; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all add column srt11_c bigint; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all add column srt12_u bigint; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all add column srt13_before_query bigint; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all add column srt14_after_query bigint; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all add column rtt_unknown bigint; diff --git a/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_906.sql b/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_906.sql index a720071e5..518497b40 100644 --- a/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_906.sql +++ b/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_906.sql @@ -13,3 +13,1130 @@ CREATE FUNCTION pg_catalog.ss_txnstatus_cache_stat( RETURNS SETOF record LANGUAGE internal STABLE NOT FENCED NOT SHIPPABLE ROWS 100 AS 'ss_txnstatus_cache_stat'; + +-- those type of view will recreate. +-- openGauss=# select table_schema, table_name,column_name from information_schema.columns where column_name = 'db_time'; +-- table_schema | table_name | column_name +-- --------------+-------------------------------+------------- +-- pg_catalog | gs_wlm_session_query_info_all | db_time -- table +-- pg_catalog | gs_wlm_session_info_all | db_time -- view +-- dbe_perf | statement | db_time -- view +-- pg_catalog | statement_history | db_time -- unlogged table +-- dbe_perf | statement_history | db_time -- view +-- dbe_perf | summary_statement | db_time -- view +-- dbe_perf | gs_slow_query_info | db_time -- view +-- dbe_perf | gs_slow_query_history | db_time -- view +-- dbe_perf | global_slow_query_history | db_time -- view +-- dbe_perf | global_slow_query_info | db_time -- view +-- those proc will recreated. +-- openGauss=# select proname, pronamespace from pg_proc where proargnames @> array['db_time']; +-- proname | pronamespace +-- ----------------------------------+-------------- +-- get_instr_unique_sql | 11 +-- pg_stat_get_wlm_session_info | 11 +-- standby_statement_history | 4988 +-- standby_statement_history | 4988 +-- get_global_full_sql_by_timestamp | 4988 +-- get_global_slow_sql_by_timestamp | 4988 + +/* We process history related tables、views and function now. */ +DROP VIEW IF EXISTS DBE_PERF.statement CASCADE; +DROP VIEW IF EXISTS DBE_PERF.summary_statement CASCADE; +DROP VIEW IF EXISTS DBE_PERF.statement_history CASCADE; +DROP FUNCTION IF EXISTS pg_catalog.get_instr_unique_sql() CASCADE; +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 5702; +CREATE FUNCTION pg_catalog.get_instr_unique_sql +( + OUT node_name name, + OUT node_id integer, + OUT user_name name, + OUT user_id oid, + OUT unique_sql_id bigint, + OUT query text, + OUT n_calls bigint, + OUT min_elapse_time bigint, + OUT max_elapse_time bigint, + OUT total_elapse_time bigint, + OUT n_returned_rows bigint, + OUT n_tuples_fetched bigint, + OUT n_tuples_returned bigint, + OUT n_tuples_inserted bigint, + OUT n_tuples_updated bigint, + OUT n_tuples_deleted bigint, + OUT n_blocks_fetched bigint, + OUT n_blocks_hit bigint, + OUT n_soft_parse bigint, + OUT n_hard_parse bigint, + OUT db_time bigint, + OUT cpu_time bigint, + OUT execution_time bigint, + OUT parse_time bigint, + OUT plan_time bigint, + OUT rewrite_time bigint, + OUT pl_execution_time bigint, + OUT pl_compilation_time bigint, + OUT data_io_time bigint, + OUT net_send_info text, + Out net_recv_info text, + OUT net_stream_send_info text, + OUT net_stream_recv_info text, + OUT last_updated timestamp with time zone, + OUT sort_count bigint, + OUT sort_time bigint, + OUT sort_mem_used bigint, + OUT sort_spill_count bigint, + OUT sort_spill_size bigint, + OUT hash_count bigint, + OUT hash_time bigint, + OUT hash_mem_used bigint, + OUT hash_spill_count bigint, + OUT hash_spill_size bigint, + OUT net_send_time bigint, + OUT srt1_q bigint, + OUT srt2_simple_query bigint, + OUT srt3_analyze_rewrite bigint, + OUT srt4_plan_query bigint, + OUT srt5_light_query bigint, + OUT srt6_p bigint, + OUT srt7_b bigint, + OUT srt8_e bigint, + OUT srt9_d bigint, + OUT srt10_s bigint, + OUT srt11_c bigint, + OUT srt12_u bigint, + OUT srt13_before_query bigint, + OUT srt14_after_query bigint, + OUT rtt_unknown bigint +) +RETURNS setof record LANGUAGE INTERNAL VOLATILE NOT FENCED as 'get_instr_unique_sql'; + +DROP INDEX IF EXISTS pg_catalog.statement_history_time_idx; +DROP TABLE IF EXISTS pg_catalog.statement_history cascade; + +CREATE unlogged table IF NOT EXISTS pg_catalog.statement_history( + db_name name, + schema_name name, + origin_node integer, + user_name name, + application_name text, + client_addr text, + client_port integer, + unique_query_id bigint, + debug_query_id bigint, + query text, + start_time timestamp with time zone, + finish_time timestamp with time zone, + slow_sql_threshold bigint, + transaction_id bigint, + thread_id bigint, + session_id bigint, + n_soft_parse bigint, + n_hard_parse bigint, + query_plan text, + n_returned_rows bigint, + n_tuples_fetched bigint, + n_tuples_returned bigint, + n_tuples_inserted bigint, + n_tuples_updated bigint, + n_tuples_deleted bigint, + n_blocks_fetched bigint, + n_blocks_hit bigint, + db_time bigint, + cpu_time bigint, + execution_time bigint, + parse_time bigint, + plan_time bigint, + rewrite_time bigint, + pl_execution_time bigint, + pl_compilation_time bigint, + data_io_time bigint, + net_send_info text, + net_recv_info text, + net_stream_send_info text, + net_stream_recv_info text, + lock_count bigint, + lock_time bigint, + lock_wait_count bigint, + lock_wait_time bigint, + lock_max_count bigint, + lwlock_count bigint, + lwlock_wait_count bigint, + lwlock_time bigint, + lwlock_wait_time bigint, + details bytea, + is_slow_sql boolean, + trace_id text, + advise text, + net_send_time bigint, + srt1_q bigint, + srt2_simple_query bigint, + srt3_analyze_rewrite bigint, + srt4_plan_query bigint, + srt5_light_query bigint, + srt6_p bigint, + srt7_b bigint, + srt8_e bigint, + srt9_d bigint, + srt10_s bigint, + srt11_c bigint, + srt12_u bigint, + srt13_before_query bigint, + srt14_after_query bigint, + rtt_unknown bigint +); +REVOKE ALL on table pg_catalog.statement_history FROM public; +create index pg_catalog.statement_history_time_idx on pg_catalog.statement_history USING btree (start_time, is_slow_sql); + +DROP FUNCTION IF EXISTS dbe_perf.standby_statement_history(boolean); +DROP FUNCTION IF EXISTS dbe_perf.standby_statement_history(boolean, timestamp with time zone[]); +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 3118; +CREATE FUNCTION dbe_perf.standby_statement_history( + IN only_slow boolean, + OUT db_name name, + OUT schema_name name, + OUT origin_node integer, + OUT user_name name, + OUT application_name text, + OUT client_addr text, + OUT client_port integer, + OUT unique_query_id bigint, + OUT debug_query_id bigint, + OUT query text, + OUT start_time timestamp with time zone, + OUT finish_time timestamp with time zone, + OUT slow_sql_threshold bigint, + OUT transaction_id bigint, + OUT thread_id bigint, + OUT session_id bigint, + OUT n_soft_parse bigint, + OUT n_hard_parse bigint, + OUT query_plan text, + OUT n_returned_rows bigint, + OUT n_tuples_fetched bigint, + OUT n_tuples_returned bigint, + OUT n_tuples_inserted bigint, + OUT n_tuples_updated bigint, + OUT n_tuples_deleted bigint, + OUT n_blocks_fetched bigint, + OUT n_blocks_hit bigint, + OUT db_time bigint, + OUT cpu_time bigint, + OUT execution_time bigint, + OUT parse_time bigint, + OUT plan_time bigint, + OUT rewrite_time bigint, + OUT pl_execution_time bigint, + OUT pl_compilation_time bigint, + OUT data_io_time bigint, + OUT net_send_info text, + OUT net_recv_info text, + OUT net_stream_send_info text, + OUT net_stream_recv_info text, + OUT lock_count bigint, + OUT lock_time bigint, + OUT lock_wait_count bigint, + OUT lock_wait_time bigint, + OUT lock_max_count bigint, + OUT lwlock_count bigint, + OUT lwlock_wait_count bigint, + OUT lwlock_time bigint, + OUT lwlock_wait_time bigint, + OUT details bytea, + OUT is_slow_sql boolean, + OUT trace_id text, + OUT advise text, + OUT net_send_time bigint, + OUT srt1_q bigint, + OUT srt2_simple_query bigint, + OUT srt3_analyze_rewrite bigint, + OUT srt4_plan_query bigint, + OUT srt5_light_query bigint, + OUT srt6_p bigint, + OUT srt7_b bigint, + OUT srt8_e bigint, + OUT srt9_d bigint, + OUT srt10_s bigint, + OUT srt11_c bigint, + OUT srt12_u bigint, + OUT srt13_before_query bigint, + OUT srt14_after_query bigint, + OUT rtt_unknown bigint) +RETURNS SETOF record NOT FENCED NOT SHIPPABLE ROWS 10000 +LANGUAGE internal AS $function$standby_statement_history_1v$function$; + +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 3119; +CREATE FUNCTION dbe_perf.standby_statement_history( + IN only_slow boolean, + VARIADIC finish_time timestamp with time zone[], + OUT db_name name, + OUT schema_name name, + OUT origin_node integer, + OUT user_name name, + OUT application_name text, + OUT client_addr text, + OUT client_port integer, + OUT unique_query_id bigint, + OUT debug_query_id bigint, + OUT query text, + OUT start_time timestamp with time zone, + OUT finish_time timestamp with time zone, + OUT slow_sql_threshold bigint, + OUT transaction_id bigint, + OUT thread_id bigint, + OUT session_id bigint, + OUT n_soft_parse bigint, + OUT n_hard_parse bigint, + OUT query_plan text, + OUT n_returned_rows bigint, + OUT n_tuples_fetched bigint, + OUT n_tuples_returned bigint, + OUT n_tuples_inserted bigint, + OUT n_tuples_updated bigint, + OUT n_tuples_deleted bigint, + OUT n_blocks_fetched bigint, + OUT n_blocks_hit bigint, + OUT db_time bigint, + OUT cpu_time bigint, + OUT execution_time bigint, + OUT parse_time bigint, + OUT plan_time bigint, + OUT rewrite_time bigint, + OUT pl_execution_time bigint, + OUT pl_compilation_time bigint, + OUT data_io_time bigint, + OUT net_send_info text, + OUT net_recv_info text, + OUT net_stream_send_info text, + OUT net_stream_recv_info text, + OUT lock_count bigint, + OUT lock_time bigint, + OUT lock_wait_count bigint, + OUT lock_wait_time bigint, + OUT lock_max_count bigint, + OUT lwlock_count bigint, + OUT lwlock_wait_count bigint, + OUT lwlock_time bigint, + OUT lwlock_wait_time bigint, + OUT details bytea, + OUT is_slow_sql boolean, + OUT net_send_time bigint, + OUT srt1_q bigint, + OUT srt2_simple_query bigint, + OUT srt3_analyze_rewrite bigint, + OUT srt4_plan_query bigint, + OUT srt5_light_query bigint, + OUT srt6_p bigint, + OUT srt7_b bigint, + OUT srt8_e bigint, + OUT srt9_d bigint, + OUT srt10_s bigint, + OUT srt11_c bigint, + OUT srt12_u bigint, + OUT srt13_before_query bigint, + OUT srt14_after_query bigint, + OUT rtt_unknown bigint) +RETURNS SETOF record NOT FENCED NOT SHIPPABLE ROWS 10000 +LANGUAGE internal AS $function$standby_statement_history$function$; + +CREATE VIEW DBE_PERF.statement AS + SELECT * FROM get_instr_unique_sql(); + +DROP FUNCTION IF EXISTS dbe_perf.get_summary_statement() cascade; +CREATE OR REPLACE FUNCTION dbe_perf.get_summary_statement() +RETURNS setof dbe_perf.statement +AS $$ +DECLARE + row_data dbe_perf.statement%rowtype; + row_name record; + query_str text; + query_str_nodes text; + BEGIN + --Get all the node names + query_str_nodes := 'select * from dbe_perf.node_name'; + FOR row_name IN EXECUTE(query_str_nodes) LOOP + query_str := 'SELECT * FROM dbe_perf.statement'; + FOR row_data IN EXECUTE(query_str) LOOP + return next row_data; + END LOOP; + END LOOP; + return; + END; $$ +LANGUAGE 'plpgsql' NOT FENCED; + +CREATE VIEW DBE_PERF.summary_statement AS + SELECT * FROM DBE_PERF.get_summary_statement(); + +CREATE VIEW DBE_PERF.statement_history AS + select * from pg_catalog.statement_history; + +DROP FUNCTION IF EXISTS dbe_perf.get_global_full_sql_by_timestamp(timestamp with time zone, timestamp with time zone); +CREATE OR REPLACE FUNCTION dbe_perf.get_global_full_sql_by_timestamp + (in start_timestamp timestamp with time zone, + in end_timestamp timestamp with time zone, + OUT node_name name, + OUT db_name name, + OUT schema_name name, + OUT origin_node integer, + OUT user_name name, + OUT application_name text, + OUT client_addr text, + OUT client_port integer, + OUT unique_query_id bigint, + OUT debug_query_id bigint, + OUT query text, + OUT start_time timestamp with time zone, + OUT finish_time timestamp with time zone, + OUT slow_sql_threshold bigint, + OUT transaction_id bigint, + OUT thread_id bigint, + OUT session_id bigint, + OUT n_soft_parse bigint, + OUT n_hard_parse bigint, + OUT query_plan text, + OUT n_returned_rows bigint, + OUT n_tuples_fetched bigint, + OUT n_tuples_returned bigint, + OUT n_tuples_inserted bigint, + OUT n_tuples_updated bigint, + OUT n_tuples_deleted bigint, + OUT n_blocks_fetched bigint, + OUT n_blocks_hit bigint, + OUT db_time bigint, + OUT cpu_time bigint, + OUT execution_time bigint, + OUT parse_time bigint, + OUT plan_time bigint, + OUT rewrite_time bigint, + OUT pl_execution_time bigint, + OUT pl_compilation_time bigint, + OUT data_io_time bigint, + OUT net_send_info text, + OUT net_recv_info text, + OUT net_stream_send_info text, + OUT net_stream_recv_info text, + OUT lock_count bigint, + OUT lock_time bigint, + OUT lock_wait_count bigint, + OUT lock_wait_time bigint, + OUT lock_max_count bigint, + OUT lwlock_count bigint, + OUT lwlock_wait_count bigint, + OUT lwlock_time bigint, + OUT lwlock_wait_time bigint, + OUT details bytea, + OUT is_slow_sql bool, + OUT trace_id text, + OUT advise text, + OUT net_send_time bigint, + OUT srt1_q bigint, + OUT srt2_simple_query bigint, + OUT srt3_analyze_rewrite bigint, + OUT srt4_plan_query bigint, + OUT srt5_light_query bigint, + OUT srt6_p bigint, + OUT srt7_b bigint, + OUT srt8_e bigint, + OUT srt9_d bigint, + OUT srt10_s bigint, + OUT srt11_c bigint, + OUT srt12_u bigint, + OUT srt13_before_query bigint, + OUT srt14_after_query bigint, + OUT rtt_unknown bigint + ) + RETURNS setof record + AS $$ + DECLARE + row_data pg_catalog.statement_history%rowtype; + row_name record; + query_str text; + -- node name + query_str_nodes text; + BEGIN + -- Get all node names(CN + master DN) + query_str_nodes := 'select * from dbe_perf.node_name'; + FOR row_name IN EXECUTE(query_str_nodes) LOOP + query_str := 'SELECT * FROM DBE_PERF.statement_history where start_time >= ''' ||$1|| ''' and start_time <= ''' || $2 || ''''; + FOR row_data IN EXECUTE(query_str) LOOP + node_name := row_name.node_name; + db_name := row_data.db_name; + schema_name := row_data.schema_name; + origin_node := row_data.origin_node; + user_name := row_data.user_name; + application_name := row_data.application_name; + client_addr := row_data.client_addr; + client_port := row_data.client_port; + unique_query_id := row_data.unique_query_id; + debug_query_id := row_data.debug_query_id; + query := row_data.query; + start_time := row_data.start_time; + finish_time := row_data.finish_time; + slow_sql_threshold := row_data.slow_sql_threshold; + transaction_id := row_data.transaction_id; + thread_id := row_data.thread_id; + session_id := row_data.session_id; + n_soft_parse := row_data.n_soft_parse; + n_hard_parse := row_data.n_hard_parse; + query_plan := row_data.query_plan; + n_returned_rows := row_data.n_returned_rows; + n_tuples_fetched := row_data.n_tuples_fetched; + n_tuples_returned := row_data.n_tuples_returned; + n_tuples_inserted := row_data.n_tuples_inserted; + n_tuples_updated := row_data.n_tuples_updated; + n_tuples_deleted := row_data.n_tuples_deleted; + n_blocks_fetched := row_data.n_blocks_fetched; + n_blocks_hit := row_data.n_blocks_hit; + db_time := row_data.db_time; + cpu_time := row_data.cpu_time; + execution_time := row_data.execution_time; + parse_time := row_data.parse_time; + plan_time := row_data.plan_time; + rewrite_time := row_data.rewrite_time; + pl_execution_time := row_data.pl_execution_time; + pl_compilation_time := row_data.pl_compilation_time; + data_io_time := row_data.data_io_time; + net_send_info := row_data.net_send_info; + net_recv_info := row_data.net_recv_info; + net_stream_send_info := row_data.net_stream_send_info; + net_stream_recv_info := row_data.net_stream_recv_info; + lock_count := row_data.lock_count; + lock_time := row_data.lock_time; + lock_wait_count := row_data.lock_wait_count; + lock_wait_time := row_data.lock_wait_time; + lock_max_count := row_data.lock_max_count; + lwlock_count := row_data.lwlock_count; + lwlock_wait_count := row_data.lwlock_wait_count; + lwlock_time := row_data.lwlock_time; + lwlock_wait_time := row_data.lwlock_wait_time; + details := row_data.details; + is_slow_sql := row_data.is_slow_sql; + trace_id := row_data.trace_id; + advise := row_data.advise; + net_send_time =row_data.net_send_time; + srt1_q := row_data.srt1_q; + srt2_simple_query := row_data.srt2_simple_query; + srt3_analyze_rewrite := row_data.srt3_analyze_rewrite; + srt4_plan_query := row_data.srt4_plan_query; + srt5_light_query := row_data.srt5_light_query; + srt6_p := row_data.srt6_p; + srt7_b := row_data.srt7_b; + srt8_e := row_data.srt8_e; + srt9_d := row_data.srt9_d; + srt10_s := row_data.srt10_s; + srt11_c := row_data.srt11_c; + srt12_u := row_data.srt12_u; + srt13_before_query := row_data.srt13_before_query; + srt14_after_query := row_data.srt14_after_query; + rtt_unknown := row_data.rtt_unknown; + return next; + END LOOP; + END LOOP; + return; + END; $$ +LANGUAGE 'plpgsql' NOT FENCED; + +DROP FUNCTION IF EXISTS dbe_perf.get_global_slow_sql_by_timestamp(timestamp with time zone, timestamp with time zone); +CREATE OR REPLACE FUNCTION DBE_PERF.get_global_slow_sql_by_timestamp + (in start_timestamp timestamp with time zone, + in end_timestamp timestamp with time zone, + OUT node_name name, + OUT db_name name, + OUT schema_name name, + OUT origin_node integer, + OUT user_name name, + OUT application_name text, + OUT client_addr text, + OUT client_port integer, + OUT unique_query_id bigint, + OUT debug_query_id bigint, + OUT query text, + OUT start_time timestamp with time zone, + OUT finish_time timestamp with time zone, + OUT slow_sql_threshold bigint, + OUT transaction_id bigint, + OUT thread_id bigint, + OUT session_id bigint, + OUT n_soft_parse bigint, + OUT n_hard_parse bigint, + OUT query_plan text, + OUT n_returned_rows bigint, + OUT n_tuples_fetched bigint, + OUT n_tuples_returned bigint, + OUT n_tuples_inserted bigint, + OUT n_tuples_updated bigint, + OUT n_tuples_deleted bigint, + OUT n_blocks_fetched bigint, + OUT n_blocks_hit bigint, + OUT db_time bigint, + OUT cpu_time bigint, + OUT execution_time bigint, + OUT parse_time bigint, + OUT plan_time bigint, + OUT rewrite_time bigint, + OUT pl_execution_time bigint, + OUT pl_compilation_time bigint, + OUT data_io_time bigint, + OUT net_send_info text, + OUT net_recv_info text, + OUT net_stream_send_info text, + OUT net_stream_recv_info text, + OUT lock_count bigint, + OUT lock_time bigint, + OUT lock_wait_count bigint, + OUT lock_wait_time bigint, + OUT lock_max_count bigint, + OUT lwlock_count bigint, + OUT lwlock_wait_count bigint, + OUT lwlock_time bigint, + OUT lwlock_wait_time bigint, + OUT details bytea, + OUT is_slow_sql bool, + OUT trace_id text, + OUT advise text, + OUT net_send_time bigint, + OUT srt1_q bigint, + OUT srt2_simple_query bigint, + OUT srt3_analyze_rewrite bigint, + OUT srt4_plan_query bigint, + OUT srt5_light_query bigint, + OUT srt6_p bigint, + OUT srt7_b bigint, + OUT srt8_e bigint, + OUT srt9_d bigint, + OUT srt10_s bigint, + OUT srt11_c bigint, + OUT srt12_u bigint, + OUT srt13_before_query bigint, + OUT srt14_after_query bigint, + OUT rtt_unknown bigint) + RETURNS setof record + AS $$ + DECLARE + row_data pg_catalog.statement_history%rowtype; + row_name record; + query_str text; + -- node name + query_str_nodes text; + BEGIN + -- Get all node names(CN + master DN) + query_str_nodes := 'select * from dbe_perf.node_name'; + FOR row_name IN EXECUTE(query_str_nodes) LOOP + query_str := 'SELECT * FROM DBE_PERF.statement_history where start_time >= ''' ||$1|| ''' and start_time <= ''' || $2 || ''' and is_slow_sql = true '; + FOR row_data IN EXECUTE(query_str) LOOP + node_name := row_name.node_name; + db_name := row_data.db_name; + schema_name := row_data.schema_name; + origin_node := row_data.origin_node; + user_name := row_data.user_name; + application_name := row_data.application_name; + client_addr := row_data.client_addr; + client_port := row_data.client_port; + unique_query_id := row_data.unique_query_id; + debug_query_id := row_data.debug_query_id; + query := row_data.query; + start_time := row_data.start_time; + finish_time := row_data.finish_time; + slow_sql_threshold := row_data.slow_sql_threshold; + transaction_id := row_data.transaction_id; + thread_id := row_data.thread_id; + session_id := row_data.session_id; + n_soft_parse := row_data.n_soft_parse; + n_hard_parse := row_data.n_hard_parse; + query_plan := row_data.query_plan; + n_returned_rows := row_data.n_returned_rows; + n_tuples_fetched := row_data.n_tuples_fetched; + n_tuples_returned := row_data.n_tuples_returned; + n_tuples_inserted := row_data.n_tuples_inserted; + n_tuples_updated := row_data.n_tuples_updated; + n_tuples_deleted := row_data.n_tuples_deleted; + n_blocks_fetched := row_data.n_blocks_fetched; + n_blocks_hit := row_data.n_blocks_hit; + db_time := row_data.db_time; + cpu_time := row_data.cpu_time; + execution_time := row_data.execution_time; + parse_time := row_data.parse_time; + plan_time := row_data.plan_time; + rewrite_time := row_data.rewrite_time; + pl_execution_time := row_data.pl_execution_time; + pl_compilation_time := row_data.pl_compilation_time; + data_io_time := row_data.data_io_time; + net_send_info := row_data.net_send_info; + net_recv_info := row_data.net_recv_info; + net_stream_send_info := row_data.net_stream_send_info; + net_stream_recv_info := row_data.net_stream_recv_info; + lock_count := row_data.lock_count; + lock_time := row_data.lock_time; + lock_wait_count := row_data.lock_wait_count; + lock_wait_time := row_data.lock_wait_time; + lock_max_count := row_data.lock_max_count; + lwlock_count := row_data.lwlock_count; + lwlock_wait_count := row_data.lwlock_wait_count; + lwlock_time := row_data.lwlock_time; + lwlock_wait_time := row_data.lwlock_wait_time; + details := row_data.details; + is_slow_sql := row_data.is_slow_sql; + trace_id := row_data.trace_id; + advise := row_data.advise; + net_send_time =row_data.net_send_time; + srt1_q := row_data.srt1_q; + srt2_simple_query := row_data.srt2_simple_query; + srt3_analyze_rewrite := row_data.srt3_analyze_rewrite; + srt4_plan_query := row_data.srt4_plan_query; + srt5_light_query := row_data.srt5_light_query; + srt6_p := row_data.srt6_p; + srt7_b := row_data.srt7_b; + srt8_e := row_data.srt8_e; + srt9_d := row_data.srt9_d; + srt10_s := row_data.srt10_s; + srt11_c := row_data.srt11_c; + srt12_u := row_data.srt12_u; + srt13_before_query := row_data.srt13_before_query; + srt14_after_query := row_data.srt14_after_query; + rtt_unknown := row_data.rtt_unknown; + return next; + END LOOP; + END LOOP; + return; + END; $$ +LANGUAGE 'plpgsql' NOT FENCED; + +/* we process wlm releates.*/ +DROP VIEW IF EXISTS pg_catalog.gs_wlm_session_info_all CASCADE; +DROP VIEW IF EXISTS pg_catalog.gs_wlm_session_info CASCADE; +DROP VIEW IF EXISTS pg_catalog.gs_wlm_session_info CASCADE; +DROP FUNCTION IF EXISTS pg_catalog.create_wlm_session_info(IN flag int) CASCADE; +DROP FUNCTION IF EXISTS pg_catalog.pg_stat_get_wlm_session_info(OID) cascade; + +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 5002; +CREATE OR REPLACE FUNCTION pg_catalog.pg_stat_get_wlm_session_info +(OID, + OUT datid oid, + OUT dbname text, + OUT schemaname text, + OUT nodename text, + OUT username text, + OUT application_name text, + OUT client_addr inet, + OUT client_hostname text, + OUT client_port integer, + OUT query_band text, + OUT block_time bigint, + OUT start_time timestamp with time zone, + OUT finish_time timestamp with time zone, + OUT duration bigint, + OUT estimate_total_time bigint, + OUT status text, + OUT abort_info text, + OUT resource_pool text, + OUT control_group text, + OUT estimate_memory integer, + OUT min_peak_memory integer, + OUT max_peak_memory integer, + OUT average_peak_memory integer, + OUT memory_skew_percent integer, + OUT spill_info text, + OUT min_spill_size integer, + OUT max_spill_size integer, + OUT average_spill_size integer, + OUT spill_skew_percent integer, + OUT min_dn_time bigint, + OUT max_dn_time bigint, + OUT average_dn_time bigint, + OUT dntime_skew_percent integer, + OUT min_cpu_time bigint, + OUT max_cpu_time bigint, + OUT total_cpu_time bigint, + OUT cpu_skew_percent integer, + OUT min_peak_iops integer, + OUT max_peak_iops integer, + OUT average_peak_iops integer, + OUT iops_skew_percent integer, + OUT warning text, + OUT queryid bigint, + OUT query text, + OUT query_plan text, + OUT node_group text, + OUT cpu_top1_node_name text, + OUT cpu_top2_node_name text, + OUT cpu_top3_node_name text, + OUT cpu_top4_node_name text, + OUT cpu_top5_node_name text, + OUT mem_top1_node_name text, + OUT mem_top2_node_name text, + OUT mem_top3_node_name text, + OUT mem_top4_node_name text, + OUT mem_top5_node_name text, + OUT cpu_top1_value bigint, + OUT cpu_top2_value bigint, + OUT cpu_top3_value bigint, + OUT cpu_top4_value bigint, + OUT cpu_top5_value bigint, + OUT mem_top1_value bigint, + OUT mem_top2_value bigint, + OUT mem_top3_value bigint, + OUT mem_top4_value bigint, + OUT mem_top5_value bigint, + OUT top_mem_dn text, + OUT top_cpu_dn text, + OUT n_returned_rows bigint, + OUT n_tuples_fetched bigint, + OUT n_tuples_returned bigint, + OUT n_tuples_inserted bigint, + OUT n_tuples_updated bigint, + OUT n_tuples_deleted bigint, + OUT n_blocks_fetched bigint, + OUT n_blocks_hit bigint, + OUT db_time bigint, + OUT cpu_time bigint, + OUT execution_time bigint, + OUT parse_time bigint, + OUT plan_time bigint, + OUT rewrite_time bigint, + OUT pl_execution_time bigint, + OUT pl_compilation_time bigint, + OUT net_send_time bigint, + OUT data_io_time bigint, + OUT is_slow_query bigint, + OUT srt1_q bigint, + OUT srt2_simple_query bigint, + OUT srt3_analyze_rewrite bigint, + OUT srt4_plan_query bigint, + OUT srt5_light_query bigint, + OUT srt6_p bigint, + OUT srt7_b bigint, + OUT srt8_e bigint, + OUT srt9_d bigint, + OUT srt10_s bigint, + OUT srt11_c bigint, + OUT srt12_u bigint, + OUT srt13_before_query bigint, + OUT srt14_after_query bigint, + OUT rtt_unknown bigint + ) RETURNS setof record LANGUAGE INTERNAL VOLATILE NOT FENCED as 'pg_stat_get_wlm_session_info'; + + +CREATE VIEW pg_catalog.gs_wlm_session_info_all AS +SELECT * FROM pg_catalog.pg_stat_get_wlm_session_info(0); + +--process wlm_session info +CREATE VIEW pg_catalog.gs_wlm_session_info AS +SELECT + S.datid, + S.dbname, + S.schemaname, + S.nodename, + S.username, + S.application_name, + S.client_addr, + S.client_hostname, + S.client_port, + S.query_band, + S.block_time, + S.start_time, + S.finish_time, + S.duration, + S.estimate_total_time, + S.status, + S.abort_info, + S.resource_pool, + S.control_group, + S.estimate_memory, + S.min_peak_memory, + S.max_peak_memory, + S.average_peak_memory, + S.memory_skew_percent, + S.spill_info, + S.min_spill_size, + S.max_spill_size, + S.average_spill_size, + S.spill_skew_percent, + S.min_dn_time, + S.max_dn_time, + S.average_dn_time, + S.dntime_skew_percent, + S.min_cpu_time, + S.max_cpu_time, + S.total_cpu_time, + S.cpu_skew_percent, + S.min_peak_iops, + S.max_peak_iops, + S.average_peak_iops, + S.iops_skew_percent, + S.warning, + S.queryid, + S.query, + S.query_plan, + S.node_group, + S.cpu_top1_node_name, + S.cpu_top2_node_name, + S.cpu_top3_node_name, + S.cpu_top4_node_name, + S.cpu_top5_node_name, + S.mem_top1_node_name, + S.mem_top2_node_name, + S.mem_top3_node_name, + S.mem_top4_node_name, + S.mem_top5_node_name, + S.cpu_top1_value, + S.cpu_top2_value, + S.cpu_top3_value, + S.cpu_top4_value, + S.cpu_top5_value, + S.mem_top1_value, + S.mem_top2_value, + S.mem_top3_value, + S.mem_top4_value, + S.mem_top5_value, + S.top_mem_dn, + S.top_cpu_dn +FROM gs_wlm_session_query_info_all S; + +CREATE VIEW pg_catalog.gs_wlm_session_history AS +SELECT + S.datid, + S.dbname, + S.schemaname, + S.nodename, + S.username, + S.application_name, + S.client_addr, + S.client_hostname, + S.client_port, + S.query_band, + S.block_time, + S.start_time, + S.finish_time, + S.duration, + S.estimate_total_time, + S.status, + S.abort_info, + S.resource_pool, + S.control_group, + S.estimate_memory, + S.min_peak_memory, + S.max_peak_memory, + S.average_peak_memory, + S.memory_skew_percent, + S.spill_info, + S.min_spill_size, + S.max_spill_size, + S.average_spill_size, + S.spill_skew_percent, + S.min_dn_time, + S.max_dn_time, + S.average_dn_time, + S.dntime_skew_percent, + S.min_cpu_time, + S.max_cpu_time, + S.total_cpu_time, + S.cpu_skew_percent, + S.min_peak_iops, + S.max_peak_iops, + S.average_peak_iops, + S.iops_skew_percent, + S.warning, + S.queryid, + S.query, + S.query_plan, + S.node_group, + S.cpu_top1_node_name, + S.cpu_top2_node_name, + S.cpu_top3_node_name, + S.cpu_top4_node_name, + S.cpu_top5_node_name, + S.mem_top1_node_name, + S.mem_top2_node_name, + S.mem_top3_node_name, + S.mem_top4_node_name, + S.mem_top5_node_name, + S.cpu_top1_value, + S.cpu_top2_value, + S.cpu_top3_value, + S.cpu_top4_value, + S.cpu_top5_value, + S.mem_top1_value, + S.mem_top2_value, + S.mem_top3_value, + S.mem_top4_value, + S.mem_top5_value, + S.top_mem_dn, + S.top_cpu_dn +FROM pg_catalog.gs_wlm_session_info_all S; + +CREATE OR REPLACE FUNCTION pg_catalog.create_wlm_session_info(IN flag int) +RETURNS int +AS $$ +DECLARE + query_str text; + record_cnt int; + BEGIN + record_cnt := 0; + + query_str := 'SELECT * FROM pg_catalog.pg_stat_get_wlm_session_info(1)'; + + IF flag > 0 THEN + EXECUTE 'INSERT INTO pg_catalog.gs_wlm_session_query_info_all ' || query_str; + ELSE + EXECUTE query_str; + END IF; + + RETURN record_cnt; + END; $$ +LANGUAGE plpgsql NOT FENCED; + +DROP FUNCTION IF EXISTS dbe_perf.global_slow_query_info() cascade; +DROP FUNCTION IF EXISTS dbe_perf.global_slow_query_history() cascade; +DROP VIEW IF EXISTS dbe_perf.gs_slow_query_info cascade; +DROP VIEW IF EXISTS dbe_perf.gs_slow_query_history cascade; + +CREATE OR REPLACE VIEW dbe_perf.gs_slow_query_info AS +SELECT + S.dbname, + S.schemaname, + S.nodename, + S.username, + S.queryid, + S.query, + S.start_time, + S.finish_time, + S.duration, + S.query_plan, + S.n_returned_rows, + S.n_tuples_fetched, + S.n_tuples_returned, + S.n_tuples_inserted, + S.n_tuples_updated, + S.n_tuples_deleted, + S.n_blocks_fetched, + S.n_blocks_hit, + S.db_time, + S.cpu_time, + S.execution_time, + S.parse_time, + S.plan_time, + S.rewrite_time, + S.pl_execution_time, + S.pl_compilation_time, + S.net_send_time, + S.data_io_time, + S.srt1_q, + S.srt2_simple_query, + S.srt3_analyze_rewrite, + S.srt4_plan_query, + S.srt5_light_query, + S.srt6_p, + S.srt7_b, + S.srt8_e, + S.srt9_d, + S.srt10_s, + S.srt11_c, + S.srt12_u, + S.srt13_before_query, + S.srt14_after_query, + S.rtt_unknown +FROM gs_wlm_session_query_info_all S where S.is_slow_query = 1; + +CREATE OR REPLACE VIEW dbe_perf.gs_slow_query_history AS +SELECT + S.dbname, + S.schemaname, + S.nodename, + S.username, + S.queryid, + S.query, + S.start_time, + S.finish_time, + S.duration, + S.query_plan, + S.n_returned_rows, + S.n_tuples_fetched, + S.n_tuples_returned, + S.n_tuples_inserted, + S.n_tuples_updated, + S.n_tuples_deleted, + S.n_blocks_fetched, + S.n_blocks_hit, + S.db_time, + S.cpu_time, + S.execution_time, + S.parse_time, + S.plan_time, + S.rewrite_time, + S.pl_execution_time, + S.pl_compilation_time, + S.net_send_time, + S.data_io_time, + S.srt1_q, + S.srt2_simple_query, + S.srt3_analyze_rewrite, + S.srt4_plan_query, + S.srt5_light_query, + S.srt6_p, + S.srt7_b, + S.srt8_e, + S.srt9_d, + S.srt10_s, + S.srt11_c, + S.srt12_u, + S.srt13_before_query, + S.srt14_after_query, + S.rtt_unknown +FROM pg_catalog.pg_stat_get_wlm_session_info(0) S where S.is_slow_query = 1; + +CREATE OR REPLACE FUNCTION dbe_perf.global_slow_query_history +RETURNS setof dbe_perf.gs_slow_query_history +AS $$ +DECLARE + row_data dbe_perf.gs_slow_query_history%rowtype; + row_name record; + query_str text; + query_str_nodes text; + BEGIN + --Get all the node names + query_str_nodes := 'SELECT node_name FROM pgxc_node WHERE node_type=''C'' AND nodeis_active = true'; + FOR row_name IN EXECUTE(query_str_nodes) LOOP + query_str := 'EXECUTE DIRECT ON (' || row_name.node_name || ') ''SELECT * FROM dbe_perf.gs_slow_query_history'''; + FOR row_data IN EXECUTE(query_str) LOOP + return next row_data; + END LOOP; + END LOOP; + return; + END; $$ +LANGUAGE 'plpgsql' NOT FENCED; + +CREATE OR REPLACE FUNCTION dbe_perf.global_slow_query_info() +RETURNS setof dbe_perf.gs_slow_query_info +AS $$ +DECLARE + row_data dbe_perf.gs_slow_query_info%rowtype; + row_name record; + query_str text; + query_str_nodes text; + BEGIN + --Get all the node names + query_str_nodes := 'SELECT node_name FROM pgxc_node WHERE node_type=''C'' AND nodeis_active = true'; + FOR row_name IN EXECUTE(query_str_nodes) LOOP + query_str := 'EXECUTE DIRECT ON (' || row_name.node_name || ') ''SELECT * FROM dbe_perf.gs_slow_query_info'''; + FOR row_data IN EXECUTE(query_str) LOOP + return next row_data; + END LOOP; + END LOOP; + return; + END; $$ +LANGUAGE 'plpgsql' NOT FENCED; + +CREATE OR REPLACE VIEW DBE_PERF.global_slow_query_history AS +SELECT * FROM DBE_PERF.global_slow_query_history(); + +CREATE OR REPLACE VIEW DBE_PERF.global_slow_query_info AS +SELECT * FROM DBE_PERF.global_slow_query_info(); + +GRANT SELECT ON TABLE pg_catalog.gs_wlm_session_query_info_all TO PUBLIC; +GRANT SELECT ON TABLE pg_catalog.gs_wlm_session_history TO PUBLIC; +GRANT SELECT ON TABLE pg_catalog.gs_wlm_session_info TO PUBLIC; +GRANT SELECT ON TABLE pg_catalog.gs_wlm_session_info_all TO PUBLIC; +GRANT SELECT ON TABLE DBE_PERF.STATEMENT TO PUBLIC; +GRANT SELECT ON TABLE DBE_PERF.summary_statement TO PUBLIC; +GRANT SELECT ON TABLE DBE_PERF.statement_history TO PUBLIC; +GRANT SELECT ON TABLE DBE_PERF.global_slow_query_history TO PUBLIC; +GRANT SELECT ON TABLE DBE_PERF.global_slow_query_info TO PUBLIC; \ No newline at end of file diff --git a/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade_catalog_otherdb_92_906.sql b/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade_catalog_otherdb_92_906.sql new file mode 100644 index 000000000..51a105b0b --- /dev/null +++ b/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade_catalog_otherdb_92_906.sql @@ -0,0 +1,22 @@ +-- this table will be add column +-- openGauss=# select table_schema, table_name,column_name from information_schema.columns where column_name = 'db_time'; +-- table_schema | table_name | column_name +-- --------------+-------------------------------+------------- +-- pg_catalog | gs_wlm_session_query_info_all | db_time --table +-- pg_catalog | statement_history | db_time --unlogged table, update in post-upgrade + +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all add column srt1_q bigint; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all add column srt2_simple_query bigint; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all add column srt3_analyze_rewrite bigint; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all add column srt4_plan_query bigint; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all add column srt5_light_query bigint; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all add column srt6_p bigint; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all add column srt7_b bigint; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all add column srt8_e bigint; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all add column srt9_d bigint; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all add column srt10_s bigint; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all add column srt11_c bigint; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all add column srt12_u bigint; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all add column srt13_before_query bigint; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all add column srt14_after_query bigint; +ALTER TABLE pg_catalog.gs_wlm_session_query_info_all add column rtt_unknown bigint; diff --git a/src/include/knl/knl_guc/knl_session_attr_common.h b/src/include/knl/knl_guc/knl_session_attr_common.h index a7e03fe6e..fa0cb7ee6 100644 --- a/src/include/knl/knl_guc/knl_session_attr_common.h +++ b/src/include/knl/knl_guc/knl_session_attr_common.h @@ -244,6 +244,7 @@ typedef struct knl_session_attr_common { bool enable_indexscan_optimization; char* delimiter_name; bool b_compatibility_user_host_auth; + int time_record_level; } knl_session_attr_common; #endif /* SRC_INCLUDE_KNL_KNL_SESSION_ATTR_COMMON_H_ */ diff --git a/src/include/knl/knl_session.h b/src/include/knl/knl_session.h index 9217f7a3a..0112d3a7b 100644 --- a/src/include/knl/knl_session.h +++ b/src/include/knl/knl_session.h @@ -68,6 +68,7 @@ #include "storage/lock/lock.h" #include "utils/elog.h" #include "tcop/dest.h" +#include "og_record_time.h" typedef void (*pg_on_exit_callback)(int code, Datum arg); @@ -1755,6 +1756,8 @@ typedef struct knl_u_stat_context { bool isTopLevelPlSql; int64* localTimeInfoArray; uint64* localNetInfo; + // use to record all use time in multi thread. + void* og_record_stat; MemoryContext pgStatLocalContext; MemoryContext pgStatCollectThdStatusContext; diff --git a/src/include/knl/knl_thread.h b/src/include/knl/knl_thread.h index fee147d2b..12d8a18f9 100755 --- a/src/include/knl/knl_thread.h +++ b/src/include/knl/knl_thread.h @@ -72,6 +72,7 @@ #include "replication/worker_internal.h" #include "replication/origin.h" #include "replication/libpqsw.h" +#include "og_record_time.h" #include "catalog/pg_subscription.h" #include "port/pg_crc32c.h" #include "ddes/dms/ss_common_attr.h" diff --git a/src/include/og_record_time.h b/src/include/og_record_time.h new file mode 100644 index 000000000..05c65ff70 --- /dev/null +++ b/src/include/og_record_time.h @@ -0,0 +1,512 @@ +/* + * Copyright (c) 2023 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * --------------------------------------------------------------------------------------- + * + * IDENTIFICATION + * src/include/og_record_time.h + * + * NOTES + * Some of this code used to record sql execute time. + * + * ------------------------------------------------------------------------- + */ + +#ifndef OG_RECORD_TIME_H +#define OG_RECORD_TIME_H +#include +#include +#include +#include "og_record_time_rely.h" +#include "knl/knl_session.h" + +typedef enum TimeInfoType { + DB_TIME = 0, /*total elapsed time while dealing user command.*/ + CPU_TIME, /*total cpu time used while dealing user command.*/ + + /*statistics of specific execution stage.*/ + EXECUTION_TIME, /*total elapsed time of execution stage.*/ + PARSE_TIME, /*total elapsed time of parse stage.*/ + PLAN_TIME, /*total elapsed time of plan stage.*/ + REWRITE_TIME, /*total elapsed time of rewrite stage.*/ + + /*statistics for plpgsql especially*/ + PL_EXECUTION_TIME, /*total elapsed time of plpgsql exection.*/ + PL_COMPILATION_TIME, /*total elapsed time of plpgsql compilation.*/ + + NET_SEND_TIME, + DATA_IO_TIME, + SRT1_Q, + SRT2_SIMPLE_QUERY, + SRT3_ANALYZE_REWRITE, + SRT4_PLAN_QUERY, + SRT5_LIGHT_QUERY, + SRT6_P, + SRT7_B, + SRT8_E, + SRT9_D, + SRT10_S, + SRT11_C, + SRT12_U, + SRT13_BEFORE_QUERY, + SRT14_AFTER_QUERY, + RTT_UNKNOWN, + TOTAL_TIME_INFO_TYPES +} TimeInfoType; + +// some procedure use old postion, so we define this. +const TimeInfoType TOTAL_TIME_INFO_TYPES_P1 = SRT1_Q; + +typedef enum NetInfoType { + NET_SEND_TIMES, + NET_SEND_N_CALLS, + NET_SEND_SIZE, + + NET_RECV_TIMES, + NET_RECV_N_CALLS, + NET_RECV_SIZE, + + NET_STREAM_SEND_TIMES, + NET_STREAM_SEND_N_CALLS, + NET_STREAM_SEND_SIZE, + + NET_STREAM_RECV_TIMES, + NET_STREAM_RECV_N_CALLS, + NET_STREAM_RECV_SIZE, + + TOTAL_NET_INFO_TYPES +} NetInfoType; + +// this for easy add new record type for debug. +typedef enum SelfRecordType { + SRT_ALL +} SelfRecordType; +// the type of record time +typedef enum RecordTimeType { + TIME_INFO = 0, + NET_INFO, + SELF_INFO +} RecordTimeType; + +const int TOTAL_RECORD_TYPES = TOTAL_TIME_INFO_TYPES + TOTAL_NET_INFO_TYPES + SRT_ALL; +// if left record time more than this, will print flag info. +const int64 DEFAULT_DB_TIME_BASELINE = 10; //ms +// the OgTimeDataStack depth +const int DEFAULT_TIME_DATA_STACK_DEPTH = 100; +// the default format length +const int DEFAULT_FORMAT_LENGTH = 1024; +const int INVALID_DEPTH = -1; +// the type of record. +class RecordType; +// the time base slice of type +class OgTimeDataVo; +// the time base vo format helper +class OgTimeDataFormatHelper; +// the time data vo stack +class OgTimeDataStack; +// the stat statics class +class OgRecordStat; + +#define FORMAT_VO(vo) (OgTimeDataFormatHelper().format(vo)) + +#ifdef _cplusplus +extern "C" { +#endif +extern const char* TimeInfoTypeName[]; +/** + * Get record stat instance, must be already init session before use it! + * */ +OgRecordStat* og_get_record_stat(); + +/** + * Clean RecordState instance memory. + * @param code error code + * @param arg the input arg + */ +void og_record_time_cleanup(int code, Datum arg); + +/** + * Reinit time record stat after set_long_jump + */ +void og_record_time_reinit(); + +/** + * Start time record, this will trigger DB_TIME begin event report. + * @return true if first started else already startted. + */ +bool og_time_record_start(); + +/** + * Stop time record, this will trigger DB_TIME end event report + * @return true if success stopped else already stopped. + */ +bool og_time_record_end(); + +/** + * Get if time record startted. + * @return true if stattted. + */ +bool og_time_record_is_started(); + +/** + * Get unique event id, id will continuously growing in a single statistic. + * @return the new event it. + */ +int64 og_get_time_unique_id(); + +/** + * Convert record_time to str. + * @param record_type the record type. + * @return const char* desc + */ +const char* og_record_time_type_str(const RecordType& record_type); + +/** + * Convert int pos to str + * @param pos the postion of record type + * @return const char* desc + */ +const char* og_record_time_type_str(int pos); + +#ifdef _cplusplus +} +#endif + +#define record_debug(fmt, ...) (og_get_record_stat()->logtrace(DEBUG1, fmt, ##__VA_ARGS__)) +#define record_info(fmt, ...) (og_get_record_stat()->logtrace(LOG, fmt, ##__VA_ARGS__)) +#define record_warn(fmt, ...) (og_get_record_stat()->logtrace(WARNING, fmt, ##__VA_ARGS__)) + +class RecordType { +public: + explicit RecordType(); + explicit RecordType(TimeInfoType time_info_type); + explicit RecordType(SelfRecordType self_typ); + explicit RecordType(NetInfoType net_info_type, ssize_t str_len); + /** + * Get which type of record it is. match the construct. + * @return + */ + RecordTimeType get_record_time_type() const; + + /** + * The enum type to int. + * @return the enum type order. + */ + int get_type_code() const; + + /** + * Only use in NET_INFO_TYPE. + * @return get the send net size len. + */ + ssize_t get_str_len() const; + + /** + * Match the RecordTimeType split postion. + * @return matched RecordTimeType base pos. + */ + int get_init_pos() const; + + /** + * Match the `og_record_time_type_str(int pos);` pos + * @return the pos of time_type_str + */ + int position() const; + + /** + * Is the DB_TIME type. DB_TIME can't be trigger by user. + * @return true if current if DB_TIME event + */ + bool is_root_type() const; + + bool operator==(TimeInfoType time_type) const; + bool operator!=(TimeInfoType time_type) const; + bool operator==(NetInfoType net_type) const; + bool operator!=(NetInfoType net_type) const; + bool operator==(SelfRecordType self_type) const; + bool operator!=(SelfRecordType self_type) const; +private: + RecordTimeType rtt_type; + int type_code; + ssize_t str_len; +}; + +class OgTimeDataVo { +public: + explicit OgTimeDataVo(): begin(0), end(0), depth(INVALID_DEPTH) + , record_type(RTT_UNKNOWN), other_cost(0), id(og_get_time_unique_id()) {} + explicit OgTimeDataVo(const RecordType& cur_record_type): begin(0), end(0), depth(INVALID_DEPTH) + , record_type(cur_record_type), other_cost(0), id(og_get_time_unique_id()) {} + + /** + * Get this stage total time. + * @return total record time. + */ + int64 total() const + { + return end - begin; + } + + /** + * Get this stage real time. sub child stage. + * @return + */ + int64 cost() const + { + return total() - other_cost; + } + + /** + * Update child stage cost. + * @param cost + */ + void update_other_cost(int64 cost) { + this->other_cost += cost; + } + + bool operator==(const OgTimeDataVo& compare) const { + return this->id == compare.id; + } + + bool operator!=(const OgTimeDataVo& compare) const { + return !(*this == compare); + } + // NOTICE: we will push this class to stack(some std::stack use this to create new instance), + // so must implementation this operator. + OgTimeDataVo& operator=(const OgTimeDataVo& vo) + { + if (this != &vo) { + this->begin = vo.begin; + this->end = vo.end; + this->depth = vo.depth; + this->record_type = vo.record_type; + this->other_cost = vo.other_cost; + this->id = vo.id; + } + return *this; + } + +public: + int64 begin; + int64 end; + int depth; + RecordType record_type; + int64 other_cost; + int64 id; +}; + +class OgTimeDataFormatHelper { +public: + const char* format(const OgTimeDataVo& vo); +private: + char format_str[DEFAULT_FORMAT_LENGTH]; +}; + +class OgTimeDataStack { +public: + OgTimeDataStack(): cur_pos(-1) {} + OgTimeDataVo& top(); + const OgTimeDataVo& top() const; + bool push(const OgTimeDataVo& vo); + void pop(); + bool empty() const; + size_t size() const; + void reset(); +private: + int cur_pos; + OgTimeDataVo data_list[DEFAULT_TIME_DATA_STACK_DEPTH]; +}; + +class OgRecordOperator { +public: + /* You must be careful when auto_record = false, it means you need ensure enter/exit pair called! + * The only situation is in loop, like in timeRecordStart/End. + * if call stack not match, some time record will discard until `OgRecordStat.reset` is called! + * We encourage this approach: OgRecordOperator _local_opt(TimeInfoType); + */ + explicit OgRecordOperator(bool auto_record); + // `NOTICE`: only TimeInfoType will record time, NetInfoType time will calculate for parent TimeInfoType stage. + // SelfRecordType only for add new stage for debug. + explicit OgRecordOperator(TimeInfoType time_info_type); + explicit OgRecordOperator(NetInfoType net_info_type); + explicit OgRecordOperator(SelfRecordType self_type); + explicit OgRecordOperator(bool auto_record, NetInfoType net_type); + explicit OgRecordOperator(bool auto_record, TimeInfoType time_type); + explicit OgRecordOperator(bool auto_record, SelfRecordType self_type); + virtual ~OgRecordOperator(); + void enter(TimeInfoType time_type=RTT_UNKNOWN); + void enter(NetInfoType net_type); + void enter(const RecordType& record_type); + void exit(TimeInfoType time_info_type=RTT_UNKNOWN); + void exit(NetInfoType net_type, ssize_t str_len); + void exit(const RecordType& record_type); + + /** + * Only TIME_INFO_TYPE will record time. this only called after exit() to report new record again. + * @param net_type NET_INTO_TYPE type + * @param str_len the send size + */ + void report_duplicate(NetInfoType net_type, ssize_t str_len); + + /** + * Get current record id. + * @return + */ + int64 get_record_id() const; + + /** + * update new record id for reuse. + */ + void update_record_id(); + + /** + * If enable report stage event. if record not start or `time_record_level` != 0 , this will false. + * @return true if need report. + */ + bool report_enable() const; +private: + void init(bool auto_record, const RecordType& record_type); +private: + OgTimeDataVo base_record; + bool auto_record; +}; + +class OgRecordStat : public BaseObject { +public: + /** + * Bind old stat memory of local_time_info and loca_net_info + * @param local_time_info the TimeInfoType memory + * @param loca_net_info the NetInfoType memory + */ + OgRecordStat(int64* local_time_info, uint64* loca_net_info); + virtual ~OgRecordStat(); + /** + * This used by DELETE_EX macro + */ + void Destroy(); + + /** + * Reset this instance. + */ + void reset(); + + /** + * Reinit after long jump + */ + void reinit(); + + /** + * Process stage event from `OgRecordOperator.enter`. + * @param data_record the record + */ + void report_start(const OgTimeDataVo& data_record); + + /** + * Process stage event from `OgRecordOperator.exit`. + * @param record the record + */ + void report_end(const OgTimeDataVo& record); + + /** + * Process start record time. this will trigger DB_TIME stage event. + * @return true if first startted. + */ + bool start_first_record_opt(); + + /** + * Process stop record time. this will trigger DB_TIME stage event. + * @return true if first stopped. + */ + bool free_first_record_opt(); + + /** + * Get the stage record time. + * @param type_info the stage + * @return record time. + */ + const int64 get_record_times(TimeInfoType type_info) const; + + /** + * Get the DB_TIME stage record time. + * @return the DB_TIME stage time. + */ + const int64 get_db_time() const; + + /** + * The root PgDataTimeVo instance, it's child_cost is equal DB_TIME stage. + * @return the root PgDataTimeVo instance. + */ + const OgTimeDataVo& get_root_time_data_vo() const; + + /** + * Get if already start record. + * @return true if already startted. + */ + bool already_start() const; + + // only for NetInfoType report. + void report_duplicate(OgTimeDataVo& record); + + /** + * Incr record depth. eveny new report stage will increment it. + * @return current depth + */ + int increment_depth(); + + /** + * Decr record depth. after record end stage will decrement it. + */ + void decrement_depth(); + + /** + * Get new unique PgDataTimeVo id. + * @return the new id. + */ + int64 get_time_unique_id(); + + /** + * Print str, only for debug use. + */ + void print_self() const; + + /** + * Get time record level for report_enable. 0-> default, all stage will report + * 1~10-> not record time. + * @return the level. + */ + int get_time_record_level() const; + + /** + * Update time record level from session. + */ + void update_time_record_level(); + + bool log_enable() const; + bool log_enable_debug() const; + void logtrace(int level, const char* fmt, ...) const; +private: + void update_record_time(const RecordType& record_type, int64 cost); + +private: + OgTimeDataStack records_stack; + OgTimeDataStack pre_records_stack; + int64* local_time_info; + uint64* local_net_info; + int depth; + int64 time_unique_id; + int64 db_time_baseline; + int time_record_level; + StringInfo log_trace_msg; + OgRecordOperator first_record_opt; + bool record_start; +}; +#endif diff --git a/src/include/og_record_time_rely.h b/src/include/og_record_time_rely.h new file mode 100644 index 000000000..43f33e3d6 --- /dev/null +++ b/src/include/og_record_time_rely.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2023 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * --------------------------------------------------------------------------------------- + * + * IDENTIFICATION + * src/include/og_record_time_rely.h + * + * NOTES + * Declare some openGauss system func or types. + * + * ------------------------------------------------------------------------- + */ + +#ifndef PG_RECORD_TIME_TMP_H +#define PG_RECORD_TIME_TMP_H +#include "postgres.h" +#include "c.h" +#include "datatype/timestamp.h" +extern TimestampTz GetCurrentTimestamp(); +void DestroyStringInfo(StringInfo str); +void ResetMemory(void* dest, size_t size); +#endif \ No newline at end of file diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 8f6049091..87f1c0ca9 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -32,6 +32,7 @@ #include "instruments/instr_event.h" #include "instruments/unique_sql_basic.h" #include "knl/knl_instance.h" +#include "og_record_time.h" /* Values for track_functions GUC variable --- order is significant! */ @@ -2474,26 +2475,6 @@ extern void DumpMemoryContext(DUMP_TYPE type); extern void getThreadMemoryDetail(Tuplestorestate* tupStore, TupleDesc tupDesc, uint32* procIdx); extern void getSharedMemoryDetail(Tuplestorestate* tupStore, TupleDesc tupDesc); -typedef enum TimeInfoType { - DB_TIME = 0, /*total elapsed time while dealing user command.*/ - CPU_TIME, /*total cpu time used while dealing user command.*/ - - /*statistics of specific execution stage.*/ - EXECUTION_TIME, /*total elapsed time of execution stage.*/ - PARSE_TIME, /*total elapsed time of parse stage.*/ - PLAN_TIME, /*total elapsed time of plan stage.*/ - REWRITE_TIME, /*total elapsed time of rewrite stage.*/ - - /*statistics for plpgsql especially*/ - PL_EXECUTION_TIME, /*total elapsed time of plpgsql exection.*/ - PL_COMPILATION_TIME, /*total elapsed time of plpgsql compilation.*/ - - NET_SEND_TIME, - DATA_IO_TIME, - - TOTAL_TIME_INFO_TYPES -} TimeInfoType; - typedef struct SessionTimeEntry { /* *protect the rest part of the entry. @@ -2527,39 +2508,37 @@ typedef struct SessionTimeEntry { #define SessionTimeArraySize (BackendStatusArray_size) -#define PGSTAT_INIT_TIME_RECORD() int64 startTime = 0; +#define PGSTAT_INIT_TIME_RECORD() OgRecordOperator og_record_opt(false); #define PGSTAT_START_TIME_RECORD() \ do { \ if (t_thrd.shemem_ptr_cxt.mySessionTimeEntry) \ - startTime = GetCurrentTimestamp(); \ + og_record_opt.enter(); \ } while (0) #define PGSTAT_END_TIME_RECORD(stage) \ do { \ if (t_thrd.shemem_ptr_cxt.mySessionTimeEntry) \ - u_sess->stat_cxt.localTimeInfoArray[stage] += GetCurrentTimestamp() - startTime; \ + og_record_opt.exit(stage); \ } while (0) #define PGSTAT_START_PLSQL_TIME_RECORD() \ do { \ if (u_sess->stat_cxt.isTopLevelPlSql && t_thrd.shemem_ptr_cxt.mySessionTimeEntry) { \ - startTime = GetCurrentTimestamp(); \ u_sess->stat_cxt.isTopLevelPlSql = false; \ needRecord = true; \ + og_record_opt.enter(); \ } \ } while (0) #define PGSTAT_END_PLSQL_TIME_RECORD(stage) \ do { \ if (needRecord == true && t_thrd.shemem_ptr_cxt.mySessionTimeEntry) { \ - u_sess->stat_cxt.localTimeInfoArray[stage] += GetCurrentTimestamp() - startTime; \ u_sess->stat_cxt.isTopLevelPlSql = true; \ + og_record_opt.exit(stage); \ } \ } while (0) -extern const char* TimeInfoTypeName[TOTAL_TIME_INFO_TYPES]; - extern Size sessionTimeShmemSize(void); extern void sessionTimeShmemInit(void); @@ -2970,6 +2949,8 @@ extern void pgstat_clean_memcxt(void); extern PgBackendStatus* gs_stat_fetch_stat_beentry(int32 beid); extern void pgstat_send(void* msg, int len); extern char* GetGlobalSessionStr(GlobalSessionId globalSessionId); +void ResetMemory(void* dest, size_t size); + typedef struct PgStat_NgMemSize { int* ngmemsize; @@ -2979,60 +2960,39 @@ typedef struct PgStat_NgMemSize { uint32 allcnt; } PgStat_NgMemSize; -typedef enum NetInfoType { - NET_SEND_TIMES, - NET_SEND_N_CALLS, - NET_SEND_SIZE, - - NET_RECV_TIMES, - NET_RECV_N_CALLS, - NET_RECV_SIZE, - - NET_STREAM_SEND_TIMES, - NET_STREAM_SEND_N_CALLS, - NET_STREAM_SEND_SIZE, - - NET_STREAM_RECV_TIMES, - NET_STREAM_RECV_N_CALLS, - NET_STREAM_RECV_SIZE, - - TOTAL_NET_INFO_TYPES -} NetInfoType; - -#define END_NET_SEND_INFO(str_len) \ - do { \ - if (str_len > 0 && t_thrd.shemem_ptr_cxt.mySessionTimeEntry) { \ - u_sess->stat_cxt.localNetInfo[NET_SEND_TIMES] += GetCurrentTimestamp() - startTime; \ - u_sess->stat_cxt.localNetInfo[NET_SEND_N_CALLS]++; \ - u_sess->stat_cxt.localNetInfo[NET_SEND_SIZE] += str_len; \ - } \ +#define END_NET_SEND_INFO(str_len) \ + do { \ + if (t_thrd.shemem_ptr_cxt.mySessionTimeEntry) { \ + og_record_opt.exit(NET_SEND_TIMES, str_len); \ + } \ } while (0) - -#define END_NET_STREAM_SEND_INFO(str_len) \ - do { \ - if (str_len > 0 && t_thrd.shemem_ptr_cxt.mySessionTimeEntry) { \ - u_sess->stat_cxt.localNetInfo[NET_STREAM_SEND_TIMES] += GetCurrentTimestamp() - startTime; \ - u_sess->stat_cxt.localNetInfo[NET_STREAM_SEND_N_CALLS]++; \ - u_sess->stat_cxt.localNetInfo[NET_STREAM_SEND_SIZE] += str_len; \ - } \ +#define END_NET_SEND_INFO_DUPLICATE(str_len) \ + do { \ + if (t_thrd.shemem_ptr_cxt.mySessionTimeEntry) { \ + og_record_opt.report_duplicate(NET_SEND_TIMES, str_len); \ + } \ + } while (0) + + +#define END_NET_STREAM_SEND_INFO(str_len) \ + do { \ + if (t_thrd.shemem_ptr_cxt.mySessionTimeEntry) { \ + og_record_opt.exit(NET_STREAM_SEND_TIMES, str_len); \ + } \ } while (0) -#define END_NET_RECV_INFO(str_len) \ - do { \ - if (str_len > 0 && t_thrd.shemem_ptr_cxt.mySessionTimeEntry) { \ - u_sess->stat_cxt.localNetInfo[NET_RECV_TIMES] += GetCurrentTimestamp() - startTime; \ - u_sess->stat_cxt.localNetInfo[NET_RECV_N_CALLS]++; \ - u_sess->stat_cxt.localNetInfo[NET_RECV_SIZE] += str_len; \ - } \ +#define END_NET_RECV_INFO(str_len) \ + do { \ + if (t_thrd.shemem_ptr_cxt.mySessionTimeEntry) { \ + og_record_opt.exit(NET_RECV_TIMES, str_len); \ + } \ } while (0) -#define END_NET_STREAM_RECV_INFO(str_len) \ - do { \ - if (str_len > 0 && t_thrd.shemem_ptr_cxt.mySessionTimeEntry) { \ - u_sess->stat_cxt.localNetInfo[NET_STREAM_RECV_TIMES] += GetCurrentTimestamp() - startTime; \ - u_sess->stat_cxt.localNetInfo[NET_STREAM_RECV_N_CALLS]++; \ - u_sess->stat_cxt.localNetInfo[NET_STREAM_RECV_SIZE] += str_len; \ - } \ +#define END_NET_STREAM_RECV_INFO(str_len) \ + do { \ + if (t_thrd.shemem_ptr_cxt.mySessionTimeEntry) { \ + og_record_opt.exit(NET_STREAM_RECV_TIMES, str_len); \ + } \ } while(0) bool GetTableGstats(Oid dbid, Oid relid, Oid parentid, PgStat_StatTabEntry *tabentry); diff --git a/src/test/regress/output/recovery_2pc_tools.source b/src/test/regress/output/recovery_2pc_tools.source index 40b317af2..6b0ac7591 100644 --- a/src/test/regress/output/recovery_2pc_tools.source +++ b/src/test/regress/output/recovery_2pc_tools.source @@ -688,6 +688,7 @@ select name,vartype,unit,min_val,max_val from pg_settings where name <> 'qunit_c temp_tablespaces | string | | | thread_pool_attr | string | | | thread_pool_stream_attr | string | | | + time_record_level | integer | | 0 | 10 time_to_target_rpo | integer | | 0 | 3600 TimeZone | string | | | timezone_abbreviations | string | | | From f6d6d6dfbec32f9e196ab66885647ff9bdf29952 Mon Sep 17 00:00:00 2001 From: totaj Date: Wed, 23 Aug 2023 19:10:53 +0800 Subject: [PATCH 156/304] Fix merge into. --- src/common/backend/parser/parse_merge.cpp | 8 +++- .../expected/single_node_mergeinto.out | 46 +++++++++++++++++++ .../regress/sql/single_node_mergeinto.sql | 30 ++++++++++++ 3 files changed, 82 insertions(+), 2 deletions(-) diff --git a/src/common/backend/parser/parse_merge.cpp b/src/common/backend/parser/parse_merge.cpp index ec14efff6..394e93b48 100644 --- a/src/common/backend/parser/parse_merge.cpp +++ b/src/common/backend/parser/parse_merge.cpp @@ -128,14 +128,18 @@ static void transformMergeJoinClause(ParseState* pstate, Node* merge) foreach (cell1, relnamespace) { ParseNamespaceItem* nitem1 = (ParseNamespaceItem*)lfirst(cell1); RangeTblEntry* entry1 = nitem1->p_rte; + bool unique = true; foreach (cell2, pstate->p_relnamespace) { ParseNamespaceItem* nitem2 = (ParseNamespaceItem*)lfirst(cell2); RangeTblEntry* entry2 = nitem2->p_rte; - if (entry1->relid == entry2->relid) { - continue; + if (entry1->relid == entry2->relid && strcmp(entry1->eref->aliasname, entry2->eref->aliasname) == 0) { + unique = false; + break; } + } + if (unique) { pstate->p_relnamespace = lappend(pstate->p_relnamespace, lfirst(cell1)); } } diff --git a/src/test/regress/expected/single_node_mergeinto.out b/src/test/regress/expected/single_node_mergeinto.out index b1b3a6292..e62acabe4 100644 --- a/src/test/regress/expected/single_node_mergeinto.out +++ b/src/test/regress/expected/single_node_mergeinto.out @@ -625,6 +625,52 @@ SET WHERE alias1.h_c_w_id >= alias1.h_amount OR alias3 != 30002; +CREATE TABLE products +( +product_id INTEGER, +product_name VARCHAR2(60), +category VARCHAR2(60) +); +INSERT INTO products VALUES (1501, 'vivitar 35mm', 'electrncs'); +INSERT INTO products VALUES (1502, 'olympus is50', 'electrncs'); +INSERT INTO products VALUES (1600, 'play gym', 'toys'); +INSERT INTO products VALUES (1601, 'lamaze', 'toys'); +INSERT INTO products VALUES (1666, 'harry potter', 'dvd'); +MERGE INTO products vp +USING products np +ON (vp.product_id = np.product_id) +WHEN MATCHED THEN +UPDATE SET vp.product_name = np.product_name, vp.category = np.category WHERE vp.product_name != 'play gym' +WHEN NOT MATCHED THEN +INSERT VALUES (np.product_id, np.product_name, np.category) WHERE np.category = 'books'; +select * from products order by 1; + product_id | product_name | category +------------+--------------+----------- + 1501 | vivitar 35mm | electrncs + 1502 | olympus is50 | electrncs + 1600 | play gym | toys + 1601 | lamaze | toys + 1666 | harry potter | dvd +(5 rows) + +MERGE INTO products vp +USING products np +ON (vp.product_id = np.product_id) +WHEN MATCHED THEN +UPDATE SET vp.product_name = np.category, vp.category = np.product_name WHERE vp.product_name != 'play gym' +WHEN NOT MATCHED THEN +INSERT VALUES (np.product_id, np.product_name, np.category) WHERE np.category = 'books'; +select * from products order by 1; + product_id | product_name | category +------------+--------------+-------------- + 1501 | electrncs | vivitar 35mm + 1502 | electrncs | olympus is50 + 1600 | play gym | toys + 1601 | toys | lamaze + 1666 | dvd | harry potter +(5 rows) + +drop table products; reset current_schema; ------------------------------------------------ -- clean up diff --git a/src/test/regress/sql/single_node_mergeinto.sql b/src/test/regress/sql/single_node_mergeinto.sql index de23b0592..96ab32392 100644 --- a/src/test/regress/sql/single_node_mergeinto.sql +++ b/src/test/regress/sql/single_node_mergeinto.sql @@ -540,6 +540,36 @@ SET WHERE alias1.h_c_w_id >= alias1.h_amount OR alias3 != 30002; + +CREATE TABLE products +( +product_id INTEGER, +product_name VARCHAR2(60), +category VARCHAR2(60) +); +INSERT INTO products VALUES (1501, 'vivitar 35mm', 'electrncs'); +INSERT INTO products VALUES (1502, 'olympus is50', 'electrncs'); +INSERT INTO products VALUES (1600, 'play gym', 'toys'); +INSERT INTO products VALUES (1601, 'lamaze', 'toys'); +INSERT INTO products VALUES (1666, 'harry potter', 'dvd'); +MERGE INTO products vp +USING products np +ON (vp.product_id = np.product_id) +WHEN MATCHED THEN +UPDATE SET vp.product_name = np.product_name, vp.category = np.category WHERE vp.product_name != 'play gym' +WHEN NOT MATCHED THEN +INSERT VALUES (np.product_id, np.product_name, np.category) WHERE np.category = 'books'; +select * from products order by 1; +MERGE INTO products vp +USING products np +ON (vp.product_id = np.product_id) +WHEN MATCHED THEN +UPDATE SET vp.product_name = np.category, vp.category = np.product_name WHERE vp.product_name != 'play gym' +WHEN NOT MATCHED THEN +INSERT VALUES (np.product_id, np.product_name, np.category) WHERE np.category = 'books'; +select * from products order by 1; + +drop table products; reset current_schema; ------------------------------------------------ -- clean up From 4519358798088322371b67d6a31947df2577daf8 Mon Sep 17 00:00:00 2001 From: justbk <249396768@qq.com> Date: Thu, 24 Aug 2023 12:33:25 +0800 Subject: [PATCH 157/304] repair log ouput with format will all execute issue in db_time --- .../process/postmaster/og_record_time.cpp | 36 ++++++++++--------- src/include/og_record_time.h | 8 ++--- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/gausskernel/process/postmaster/og_record_time.cpp b/src/gausskernel/process/postmaster/og_record_time.cpp index b27b0d42b..d615aa10d 100644 --- a/src/gausskernel/process/postmaster/og_record_time.cpp +++ b/src/gausskernel/process/postmaster/og_record_time.cpp @@ -486,7 +486,7 @@ void OgRecordStat::report_start(const OgTimeDataVo& data_record) return; } records_stack.top().depth = increment_depth(); - record_debug("begin: %s", FORMAT_VO(records_stack.top())); + log_vo("begin", records_stack.top()); if (data_record.record_type.is_root_type()) { records_stack.top().begin = GetCurrentTimestamp(); } @@ -497,8 +497,8 @@ void OgRecordStat::report_end(const OgTimeDataVo& record) // assert not records_stack.is_empty() OgTimeDataVo& time_vo = records_stack.top(); if (record != time_vo) { - record_debug("find time record not match!top =%s, curr=%s", - FORMAT_VO(time_vo), FORMAT_VO(record)); + log_vo("unmatch_top:", time_vo); + log_vo("unmatch_cur:", record); return; } records_stack.pop(); @@ -510,9 +510,9 @@ void OgRecordStat::report_end(const OgTimeDataVo& record) if (time_vo.record_type.is_root_type()) { // We don't want debug time calc in root type. time_vo.end = GetCurrentTimestamp(); - record_debug("end : %s", FORMAT_VO(time_vo)); + log_vo(" end", time_vo); } else { - record_debug("end : %s", FORMAT_VO(time_vo)); + log_vo(" end", time_vo); time_vo.end = GetCurrentTimestamp(); } OgTimeDataVo& parent_time_vo = records_stack.top(); @@ -528,7 +528,7 @@ void OgRecordStat::report_end(const OgTimeDataVo& record) void OgRecordStat::report_duplicate(OgTimeDataVo& record) { Assert(record.record_type.get_record_time_type() == NET_INFO); - record_debug("duplicate : %s, str_len=%d", FORMAT_VO(record), record.record_type.get_str_len()); + log_vo("duplicate", record); if (record.end == 0) { record.end = GetCurrentTimestamp(); } @@ -555,7 +555,7 @@ void OgRecordStat::print_self() const { int64 bind_total = 0; int64 bind_data = 0; - if (!log_enable_debug()) { + if (!log_enable(DEBUG1)) { return; } for (int i = 0; i < TOTAL_RECORD_TYPES; i ++) @@ -571,10 +571,10 @@ void OgRecordStat::print_self() const } else { // nothing to do } - record_debug("name:%d %30s %ld", i, og_record_time_type_str(i), bind_data); + logtrace(DEBUG1, "name:%d %30s %ld", i, og_record_time_type_str(i), bind_data); } int64 child_cost = bind_total - get_record_times(DB_TIME) - get_record_times(CPU_TIME); - record_debug("%s,%s,diff(%lld) = total=(%lld) - child_cost(%lld)", + logtrace(DEBUG1, "%s,%s,diff(%lld) = total=(%lld) - child_cost(%lld)", get_record_times(RTT_UNKNOWN) < db_time_baseline ? "rd_yes" : "rd_no ", (child_cost == get_db_time()) ? "rd_eq " : "rd_ne", get_record_times(RTT_UNKNOWN), @@ -618,7 +618,7 @@ bool OgRecordStat::free_first_record_opt() while (records_stack.size() > 2 && (records_stack.top().id != record_id)) { records_stack.top().end = GetCurrentTimestamp(); pre_records_stack.push(records_stack.top()); - logtrace(DEBUG1, "free_head, %s", OgTimeDataFormatHelper().format(records_stack.top())); + log_vo("free_head", records_stack.top()); report_end(records_stack.top()); } first_record_opt.exit(); @@ -669,15 +669,19 @@ void OgRecordStat::update_record_time(const RecordType& record_type, int64 cost) } // flow database system log config -bool OgRecordStat::log_enable() const -{ - return u_sess != NULL && u_sess->attr.attr_common.log_statement == LOGSTMT_ALL; +bool OgRecordStat::log_enable(int level) const { + return u_sess != NULL && u_sess->attr.attr_common.log_statement == LOGSTMT_ALL + && level >= u_sess->attr.attr_common.log_min_messages; } -// flow database system log config -bool OgRecordStat::log_enable_debug() const +void OgRecordStat::log_vo(const char* tag, const OgTimeDataVo& vo) const { - return log_enable() && u_sess->attr.attr_common.log_min_messages < LOG; + if (!log_enable(DEBUG1)) { + return; + } + ereport(DEBUG1, (errmsg("record(%ld-%ld) %s %s", (uint64)this, + u_sess == NULL ? 0 : u_sess->session_id, tag, + FORMAT_VO(vo)))); } void OgRecordStat::logtrace(int level, const char* fmt, ...) const diff --git a/src/include/og_record_time.h b/src/include/og_record_time.h index 05c65ff70..103ff0076 100644 --- a/src/include/og_record_time.h +++ b/src/include/og_record_time.h @@ -182,10 +182,6 @@ const char* og_record_time_type_str(int pos); } #endif -#define record_debug(fmt, ...) (og_get_record_stat()->logtrace(DEBUG1, fmt, ##__VA_ARGS__)) -#define record_info(fmt, ...) (og_get_record_stat()->logtrace(LOG, fmt, ##__VA_ARGS__)) -#define record_warn(fmt, ...) (og_get_record_stat()->logtrace(WARNING, fmt, ##__VA_ARGS__)) - class RecordType { public: explicit RecordType(); @@ -490,8 +486,8 @@ class OgRecordStat : public BaseObject { */ void update_time_record_level(); - bool log_enable() const; - bool log_enable_debug() const; + bool log_enable(int level = LOG) const; + void log_vo(const char* tag, const OgTimeDataVo& vo) const; void logtrace(int level, const char* fmt, ...) const; private: void update_record_time(const RecordType& record_type, int64 cost); From 8d3fdacd6b35689788b1ea442de4dd44fd0802a5 Mon Sep 17 00:00:00 2001 From: vastdata-xyzr Date: Thu, 24 Aug 2023 16:09:59 +0800 Subject: [PATCH 158/304] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E7=BC=BA=E5=A4=B1?= =?UTF-8?q?=E7=9A=84=E6=96=87=E4=BB=B6=E5=85=B3=E9=97=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/tools/entab/entab.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/tools/entab/entab.cpp b/src/tools/entab/entab.cpp index 0cbfcf5b3..b71a2a172 100644 --- a/src/tools/entab/entab.cpp +++ b/src/tools/entab/entab.cpp @@ -180,6 +180,8 @@ int main(int argc, char** argv) if (fputs(out_line, stdout) == EOF) halt("PERROR: Error writing output.\n"); } + if (in_file != stdin) + fclose(in_file); } while (--argc > 0); return 0; } From f3ee5c5efa68e309231f8e1d1b14553bef45e8c7 Mon Sep 17 00:00:00 2001 From: totaj Date: Thu, 24 Aug 2023 17:38:31 +0800 Subject: [PATCH 159/304] Fix update/rollback failed with dolphin. --- build/script/aarch64_lite_list | 2 ++ build/script/aarch64_opengauss_list | 2 ++ build/script/x86_64_lite_list | 2 ++ build/script/x86_64_opengauss_list | 2 ++ .../rollback-post_catalog_maindb_92_906.sql | 26 +++++++++++++++++ .../rollback-post_catalog_otherdb_92_906.sql | 24 ++++++++++++++++ .../upgrade-post_catalog_maindb_92_906.sql | 28 ++++++++++++++++++- .../upgrade-post_catalog_otherdb_92_906.sql | 28 ++++++++++++++++++- 8 files changed, 112 insertions(+), 2 deletions(-) diff --git a/build/script/aarch64_lite_list b/build/script/aarch64_lite_list index 3d6e75783..10038ccce 100644 --- a/build/script/aarch64_lite_list +++ b/build/script/aarch64_lite_list @@ -37,8 +37,10 @@ ./share/postgresql/extension/dolphin--1.0.sql ./share/postgresql/extension/dolphin--1.0--2.0.sql ./share/postgresql/extension/dolphin--2.0--1.0.sql +./share/postgresql/extension/dolphin--2.0--2.0.1.sql ./share/postgresql/extension/dolphin--1.1--2.0.sql ./share/postgresql/extension/dolphin--2.0--1.1.sql +./share/postgresql/extension/dolphin--2.0.1--2.0.sql ./share/postgresql/extension/openGauss_expr_dolphin.ir ./share/postgresql/extension/file_fdw--1.0.sql ./share/postgresql/extension/plpgsql.control diff --git a/build/script/aarch64_opengauss_list b/build/script/aarch64_opengauss_list index 305cdd247..2904c867a 100644 --- a/build/script/aarch64_opengauss_list +++ b/build/script/aarch64_opengauss_list @@ -76,8 +76,10 @@ ./share/postgresql/extension/dolphin--1.0.sql ./share/postgresql/extension/dolphin--1.0--2.0.sql ./share/postgresql/extension/dolphin--2.0--1.0.sql +./share/postgresql/extension/dolphin--2.0--2.0.1.sql ./share/postgresql/extension/dolphin--1.1--2.0.sql ./share/postgresql/extension/dolphin--2.0--1.1.sql +./share/postgresql/extension/dolphin--2.0.1--2.0.sql ./share/postgresql/extension/openGauss_expr_dolphin.ir ./share/postgresql/extension/assessment--1.0.sql ./share/postgresql/extension/assessment.control diff --git a/build/script/x86_64_lite_list b/build/script/x86_64_lite_list index 62660db0e..b49cbef71 100644 --- a/build/script/x86_64_lite_list +++ b/build/script/x86_64_lite_list @@ -37,8 +37,10 @@ ./share/postgresql/extension/dolphin--1.0.sql ./share/postgresql/extension/dolphin--1.0--2.0.sql ./share/postgresql/extension/dolphin--2.0--1.0.sql +./share/postgresql/extension/dolphin--2.0--2.0.1.sql ./share/postgresql/extension/dolphin--1.1--2.0.sql ./share/postgresql/extension/dolphin--2.0--1.1.sql +./share/postgresql/extension/dolphin--2.0.1--2.0.sql ./share/postgresql/extension/openGauss_expr_dolphin.ir ./share/postgresql/extension/file_fdw--1.0.sql ./share/postgresql/extension/plpgsql.control diff --git a/build/script/x86_64_opengauss_list b/build/script/x86_64_opengauss_list index 69137a38f..4f1eee380 100644 --- a/build/script/x86_64_opengauss_list +++ b/build/script/x86_64_opengauss_list @@ -76,8 +76,10 @@ ./share/postgresql/extension/dolphin--1.0.sql ./share/postgresql/extension/dolphin--1.0--2.0.sql ./share/postgresql/extension/dolphin--2.0--1.0.sql +./share/postgresql/extension/dolphin--2.0--2.0.1.sql ./share/postgresql/extension/dolphin--1.1--2.0.sql ./share/postgresql/extension/dolphin--2.0--1.1.sql +./share/postgresql/extension/dolphin--2.0.1--2.0.sql ./share/postgresql/extension/openGauss_expr_dolphin.ir ./share/postgresql/extension/assessment--1.0.sql ./share/postgresql/extension/assessment.control diff --git a/src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback-post_catalog_maindb_92_906.sql b/src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback-post_catalog_maindb_92_906.sql index c03aaa2e2..9408933b6 100644 --- a/src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback-post_catalog_maindb_92_906.sql +++ b/src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback-post_catalog_maindb_92_906.sql @@ -1,3 +1,16 @@ +do $$ +DECLARE +ans boolean; +BEGIN + for ans in select case when count(*)=1 then true else false end as ans from (select extname from pg_extension where extname='dolphin' and extversion = '2.0') + LOOP + if ans = true then + ALTER EXTENSION dolphin UPDATE TO '2.0.1'; + end if; + exit; + END LOOP; +END$$; + DROP FUNCTION IF EXISTS pg_catalog.ss_txnstatus_cache_stat() CASCADE; -- those type of view and producer will recreate. @@ -954,3 +967,16 @@ GRANT SELECT ON TABLE DBE_PERF.summary_statement TO PUBLIC; GRANT SELECT ON TABLE DBE_PERF.statement_history TO PUBLIC; GRANT SELECT ON TABLE DBE_PERF.global_slow_query_history TO PUBLIC; GRANT SELECT ON TABLE DBE_PERF.global_slow_query_info TO PUBLIC; + +do $$ +DECLARE +ans boolean; +BEGIN + for ans in select case when count(*)=1 then true else false end as ans from (select extname from pg_extension where extname='dolphin' and extversion = '2.0.1') + LOOP + if ans = true then + ALTER EXTENSION dolphin UPDATE TO '2.0'; + end if; + exit; + END LOOP; +END$$; \ No newline at end of file diff --git a/src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback-post_catalog_otherdb_92_906.sql b/src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback-post_catalog_otherdb_92_906.sql index f2a9f31ac..02fd35b98 100644 --- a/src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback-post_catalog_otherdb_92_906.sql +++ b/src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback-post_catalog_otherdb_92_906.sql @@ -1,3 +1,15 @@ +do $$ +DECLARE +ans boolean; +BEGIN + for ans in select case when count(*)=1 then true else false end as ans from (select extname from pg_extension where extname='dolphin' and extversion = '2.0') + LOOP + if ans = true then + ALTER EXTENSION dolphin UPDATE TO '2.0.1'; + end if; + exit; + END LOOP; +END$$; DROP FUNCTION IF EXISTS pg_catalog.ss_txnstatus_cache_stat() CASCADE; -- those type of view and producer will recreate. @@ -955,3 +967,15 @@ GRANT SELECT ON TABLE DBE_PERF.statement_history TO PUBLIC; GRANT SELECT ON TABLE DBE_PERF.global_slow_query_history TO PUBLIC; GRANT SELECT ON TABLE DBE_PERF.global_slow_query_info TO PUBLIC; +do $$ +DECLARE +ans boolean; +BEGIN + for ans in select case when count(*)=1 then true else false end as ans from (select extname from pg_extension where extname='dolphin' and extversion = '2.0.1') + LOOP + if ans = true then + ALTER EXTENSION dolphin UPDATE TO '2.0'; + end if; + exit; + END LOOP; +END$$; \ No newline at end of file diff --git a/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_906.sql b/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_906.sql index 518497b40..74f6fce90 100644 --- a/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_906.sql +++ b/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_906.sql @@ -1,3 +1,16 @@ +do $$ +DECLARE +ans boolean; +BEGIN + for ans in select case when count(*)=1 then true else false end as ans from (select extname from pg_extension where extname='dolphin' and extversion = '2.0') + LOOP + if ans = true then + ALTER EXTENSION dolphin UPDATE TO '2.0.1'; + end if; + exit; + END LOOP; +END$$; + DROP FUNCTION IF EXISTS pg_catalog.ss_txnstatus_cache_stat() CASCADE; SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 8888; CREATE FUNCTION pg_catalog.ss_txnstatus_cache_stat( @@ -1139,4 +1152,17 @@ GRANT SELECT ON TABLE DBE_PERF.STATEMENT TO PUBLIC; GRANT SELECT ON TABLE DBE_PERF.summary_statement TO PUBLIC; GRANT SELECT ON TABLE DBE_PERF.statement_history TO PUBLIC; GRANT SELECT ON TABLE DBE_PERF.global_slow_query_history TO PUBLIC; -GRANT SELECT ON TABLE DBE_PERF.global_slow_query_info TO PUBLIC; \ No newline at end of file +GRANT SELECT ON TABLE DBE_PERF.global_slow_query_info TO PUBLIC; + +do $$ +DECLARE +ans boolean; +BEGIN + for ans in select case when count(*)=1 then true else false end as ans from (select extname from pg_extension where extname='dolphin' and extversion = '2.0.1') + LOOP + if ans = true then + ALTER EXTENSION dolphin UPDATE TO '2.0'; + end if; + exit; + END LOOP; +END$$; \ No newline at end of file diff --git a/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_906.sql b/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_906.sql index 518497b40..74f6fce90 100644 --- a/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_906.sql +++ b/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_906.sql @@ -1,3 +1,16 @@ +do $$ +DECLARE +ans boolean; +BEGIN + for ans in select case when count(*)=1 then true else false end as ans from (select extname from pg_extension where extname='dolphin' and extversion = '2.0') + LOOP + if ans = true then + ALTER EXTENSION dolphin UPDATE TO '2.0.1'; + end if; + exit; + END LOOP; +END$$; + DROP FUNCTION IF EXISTS pg_catalog.ss_txnstatus_cache_stat() CASCADE; SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 8888; CREATE FUNCTION pg_catalog.ss_txnstatus_cache_stat( @@ -1139,4 +1152,17 @@ GRANT SELECT ON TABLE DBE_PERF.STATEMENT TO PUBLIC; GRANT SELECT ON TABLE DBE_PERF.summary_statement TO PUBLIC; GRANT SELECT ON TABLE DBE_PERF.statement_history TO PUBLIC; GRANT SELECT ON TABLE DBE_PERF.global_slow_query_history TO PUBLIC; -GRANT SELECT ON TABLE DBE_PERF.global_slow_query_info TO PUBLIC; \ No newline at end of file +GRANT SELECT ON TABLE DBE_PERF.global_slow_query_info TO PUBLIC; + +do $$ +DECLARE +ans boolean; +BEGIN + for ans in select case when count(*)=1 then true else false end as ans from (select extname from pg_extension where extname='dolphin' and extversion = '2.0.1') + LOOP + if ans = true then + ALTER EXTENSION dolphin UPDATE TO '2.0'; + end if; + exit; + END LOOP; +END$$; \ No newline at end of file From 24ad69c562583beb9e9e7846b2ec68a5019a0851 Mon Sep 17 00:00:00 2001 From: movead Date: Fri, 25 Aug 2023 13:49:21 +0800 Subject: [PATCH 160/304] fix wait issue for vacuum record redo --- .../access/transam/parallel_recovery/page_redo.cpp | 7 +++++-- .../access/transam/parallel_recovery/txn_redo.cpp | 11 +++++++++-- src/include/access/parallel_recovery/txn_redo.h | 1 + 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/gausskernel/storage/access/transam/parallel_recovery/page_redo.cpp b/src/gausskernel/storage/access/transam/parallel_recovery/page_redo.cpp index de16d88d1..f2b17460a 100755 --- a/src/gausskernel/storage/access/transam/parallel_recovery/page_redo.cpp +++ b/src/gausskernel/storage/access/transam/parallel_recovery/page_redo.cpp @@ -566,6 +566,7 @@ static void wait_valid_snapshot(XLogReaderState *record) xl_heap_clean* xlrec = NULL; uint64 blockcnt = 0; XLogRecPtr cur_transed_lsn = InvalidXLogRecPtr; + XLogRecPtr txn_trying_lsn = InvalidXLogRecPtr; if(rm_id != RM_HEAP2_ID || info != XLOG_HEAP2_CLEAN) return; @@ -580,16 +581,18 @@ static void wait_valid_snapshot(XLogReaderState *record) !in_full_sync_dispatch()) { if(cur_transed_lsn == InvalidXLogRecPtr) cur_transed_lsn = getTransedTxnLsn(g_dispatcher->txnWorker); + txn_trying_lsn = getTryingTxnLsn(g_dispatcher->txnWorker); /* * Normaly, it need wait for startup thread handle xact wal records, but there be a case * that if a very old xid commit and no new xact comes then xlrec->latestRemovedXid > * t_thrd.xact_cxt.ShmemVariableCache->standbyXmin all the time. * * So if all xact record before current vacuum record finished, then avoid wait. + * And if startup go fast then here on lsn, it can avoid wait too. */ - if (cur_transed_lsn <= GetXLogReplayRecPtr(NULL)) + if (cur_transed_lsn <= GetXLogReplayRecPtr(NULL) || txn_trying_lsn >= record->EndRecPtr) return; - pg_usleep(10); + blockcnt++; if ((blockcnt & OUTPUT_WAIT_COUNT) == OUTPUT_WAIT_COUNT) { XLogRecPtr LatestReplayedRecPtr = GetXLogReplayRecPtr(NULL); diff --git a/src/gausskernel/storage/access/transam/parallel_recovery/txn_redo.cpp b/src/gausskernel/storage/access/transam/parallel_recovery/txn_redo.cpp index 141671aa1..0079b4a61 100644 --- a/src/gausskernel/storage/access/transam/parallel_recovery/txn_redo.cpp +++ b/src/gausskernel/storage/access/transam/parallel_recovery/txn_redo.cpp @@ -61,8 +61,9 @@ struct TxnRedoWorker { RedoItem *pendingTail; /* The tail of the RedoItem list. */ RedoItem *procHead; RedoItem *procTail; - XLogRecPtr dispatched_txn_lsn; - XLogRecPtr transed_txn_lsn; + XLogRecPtr dispatched_txn_lsn; /* Max lsn dispatched to txn worker*/ + XLogRecPtr transed_txn_lsn; /* Max lsn transfer to txn worker list*/ + XLogRecPtr txn_trying_lsn; /* EndPtr of trying record on txn worker*/ }; XLogRecPtr getTransedTxnLsn(TxnRedoWorker *worker) @@ -70,6 +71,11 @@ XLogRecPtr getTransedTxnLsn(TxnRedoWorker *worker) return (XLogRecPtr)pg_atomic_read_u64((volatile uint64*)&worker->transed_txn_lsn); } +XLogRecPtr getTryingTxnLsn(TxnRedoWorker *worker) +{ + return (XLogRecPtr)pg_atomic_read_u64((volatile uint64*)&worker->txn_trying_lsn); +} + TxnRedoWorker *StartTxnRedoWorker() { TxnRedoWorker *worker = (TxnRedoWorker *)palloc(sizeof(TxnRedoWorker)); @@ -267,6 +273,7 @@ void ApplyReadyTxnLogRecords(TxnRedoWorker *worker, bool forceAll) XLogReaderState *record = &item->record; XLogRecPtr lrEnd; + pg_atomic_write_u64(&worker->txn_trying_lsn, record->EndRecPtr); if (forceAll) { GetRedoStartTime(t_thrd.xlog_cxt.timeCost[TIME_COST_STEP_6]); XLogRecPtr lrRead; /* lastReplayedReadPtr */ diff --git a/src/include/access/parallel_recovery/txn_redo.h b/src/include/access/parallel_recovery/txn_redo.h index 6c1cb3fe4..3b758cea6 100644 --- a/src/include/access/parallel_recovery/txn_redo.h +++ b/src/include/access/parallel_recovery/txn_redo.h @@ -39,5 +39,6 @@ void MoveTxnItemToApplyQueue(TxnRedoWorker* worker); void DumpTxnWorker(TxnRedoWorker* txnWorker); bool IsTxnWorkerIdle(TxnRedoWorker* worker); XLogRecPtr getTransedTxnLsn(TxnRedoWorker *worker); +XLogRecPtr getTryingTxnLsn(TxnRedoWorker *worker); } #endif From 9948bb1878530e9b6687da24fcba53987d2c2bf3 Mon Sep 17 00:00:00 2001 From: justbk <249396768@qq.com> Date: Fri, 25 Aug 2023 15:02:41 +0800 Subject: [PATCH 161/304] repair WLMArbiterWorker process before main process exit will coredump --- src/common/backend/utils/error/elog.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/backend/utils/error/elog.cpp b/src/common/backend/utils/error/elog.cpp index 741774245..22454bfc2 100644 --- a/src/common/backend/utils/error/elog.cpp +++ b/src/common/backend/utils/error/elog.cpp @@ -287,7 +287,7 @@ bool errstart(int elevel, const char* filename, int lineno, const char* funcname * during which panic is not expected. */ if (AmCheckpointerProcess() || AmBackgroundWriterProcess() || AmWalReceiverWriterProcess() || - AmDataReceiverWriterProcess()) + AmDataReceiverWriterProcess() || AmWLMArbiterProcess()) elevel = FATAL; else elevel = PANIC; From a33337f1c9c06bfb57dcaf4f1b1895036cf46fb0 Mon Sep 17 00:00:00 2001 From: zhangao_za <18829237393@163.com> Date: Fri, 25 Aug 2023 16:44:45 +0800 Subject: [PATCH 162/304] =?UTF-8?q?=E5=88=A0=E9=99=A4=E8=BF=87=E5=A4=9A?= =?UTF-8?q?=E6=97=A0=E7=94=A8=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/ddes/script/dms_contrl.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/gausskernel/ddes/script/dms_contrl.sh b/src/gausskernel/ddes/script/dms_contrl.sh index 8e2ec2a95..9d02573f3 100644 --- a/src/gausskernel/ddes/script/dms_contrl.sh +++ b/src/gausskernel/ddes/script/dms_contrl.sh @@ -208,8 +208,6 @@ function Check() log "check ${GSDB_BIN} in ${GSDB_HOME} fail." exit 1 fi - - log "check gaussdb in ${GSDB_HOME} success." } # 1st step: kill database From 7d9dc642cc27b5e19fbaf79c5eae779bdd3b5e39 Mon Sep 17 00:00:00 2001 From: li-judong Date: Sat, 12 Aug 2023 18:42:21 +0800 Subject: [PATCH 163/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=A4=9A=E5=AD=97?= =?UTF-8?q?=E7=AC=A6=E9=9B=86=E5=9C=BA=E6=99=AFpbe=E7=BB=93=E6=9E=9C?= =?UTF-8?q?=E9=9B=86=E4=B8=8E=E7=9B=B4=E6=8E=A5=E6=89=A7=E8=A1=8C=E4=B8=8D?= =?UTF-8?q?=E5=90=8C=E7=9A=84=E9=97=AE=E9=A2=98=EF=BC=9B=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?PBE=E5=8F=82=E6=95=B0=E5=80=BC=E6=9C=AA=E8=BD=AC=E5=AD=97?= =?UTF-8?q?=E7=AC=A6=E9=9B=86=E7=BC=96=E7=A0=81=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/nodes/copyfuncs.cpp | 1 + src/common/backend/nodes/equalfuncs.cpp | 1 + src/common/backend/nodes/outfuncs.cpp | 3 + src/common/backend/nodes/readfuncs.cpp | 4 + src/common/backend/parser/parse_coerce.cpp | 5 +- src/common/backend/parser/parse_collate.cpp | 6 + src/common/backend/parser/parse_param.cpp | 8 +- src/common/backend/utils/cache/plancache.cpp | 14 +- src/common/backend/utils/init/globals.cpp | 3 +- src/common/backend/utils/mb/mbutils.cpp | 108 ++++++++++---- .../optimizer/commands/prepare.cpp | 11 ++ src/gausskernel/process/tcop/postgres.cpp | 54 ++++++- src/gausskernel/runtime/opfusion/opfusion.cpp | 4 + src/include/mb/pg_wchar.h | 1 + src/include/miscadmin.h | 1 + src/include/nodes/primnodes.h | 1 + src/include/parser/parse_coerce.h | 4 +- src/include/utils/plancache.h | 2 + .../regress/expected/charset_gbk_b_db.out | 139 +++++++++++++++--- .../regress/expected/charset_utf8mb4_b_db.out | 139 ++++++++++++++---- src/test/regress/sql/charset_gbk_b_db.sql | 54 +++++-- src/test/regress/sql/charset_utf8mb4_b_db.sql | 49 ++++-- 22 files changed, 494 insertions(+), 118 deletions(-) diff --git a/src/common/backend/nodes/copyfuncs.cpp b/src/common/backend/nodes/copyfuncs.cpp index c5e522bda..018742ef0 100644 --- a/src/common/backend/nodes/copyfuncs.cpp +++ b/src/common/backend/nodes/copyfuncs.cpp @@ -2563,6 +2563,7 @@ static Param* _copyParam(const Param* from) COPY_SCALAR_FIELD(tableOfIndexType); COPY_SCALAR_FIELD(recordVarTypOid); COPY_NODE_FIELD(tableOfIndexTypeList); + COPY_SCALAR_FIELD(is_bind_param); return newnode; } diff --git a/src/common/backend/nodes/equalfuncs.cpp b/src/common/backend/nodes/equalfuncs.cpp index a6b102fea..0395e0e99 100644 --- a/src/common/backend/nodes/equalfuncs.cpp +++ b/src/common/backend/nodes/equalfuncs.cpp @@ -209,6 +209,7 @@ static bool _equalParam(const Param* a, const Param* b) COMPARE_LOCATION_FIELD(location); COMPARE_SCALAR_FIELD(tableOfIndexType); COMPARE_SCALAR_FIELD(recordVarTypOid); + COMPARE_SCALAR_FIELD(is_bind_param); return true; } diff --git a/src/common/backend/nodes/outfuncs.cpp b/src/common/backend/nodes/outfuncs.cpp index 193aa86aa..3e4f8ea50 100755 --- a/src/common/backend/nodes/outfuncs.cpp +++ b/src/common/backend/nodes/outfuncs.cpp @@ -2344,6 +2344,9 @@ static void _outParam(StringInfo str, Param* node) WRITE_NODE_FIELD(tableOfIndexTypeList); } + if (t_thrd.proc->workingVersionNum >= PARAM_MARK_VERSION_NUM) { + WRITE_BOOL_FIELD(is_bind_param); + } } static void _outRownum(StringInfo str, const Rownum* node) diff --git a/src/common/backend/nodes/readfuncs.cpp b/src/common/backend/nodes/readfuncs.cpp index 254724861..6b846ca50 100755 --- a/src/common/backend/nodes/readfuncs.cpp +++ b/src/common/backend/nodes/readfuncs.cpp @@ -2142,6 +2142,10 @@ static Param* _readParam(void) { READ_NODE_FIELD(tableOfIndexTypeList); } + IF_EXIST(is_bind_param) + { + READ_BOOL_FIELD(is_bind_param); + } READ_DONE(); } diff --git a/src/common/backend/parser/parse_coerce.cpp b/src/common/backend/parser/parse_coerce.cpp index ed16ec6c5..bc616ece7 100644 --- a/src/common/backend/parser/parse_coerce.cpp +++ b/src/common/backend/parser/parse_coerce.cpp @@ -242,7 +242,8 @@ Node* coerce_to_target_type(ParseState* pstate, Node* expr, Oid exprtype, Oid ta * target_charset - desired result character set * target_type - desired result type */ -Node* coerce_to_target_charset(Node* expr, int target_charset, Oid target_type, int32 target_typmod, Oid target_collation) +Node* coerce_to_target_charset(Node* expr, int target_charset, Oid target_type, int32 target_typmod, + Oid target_collation, bool eval_const) { FuncExpr* fexpr = NULL; Node* result = NULL; @@ -294,7 +295,7 @@ Node* coerce_to_target_charset(Node* expr, int target_charset, Oid target_type, /* set collation after coerce_to_target_type */ exprSetCollation(result, target_collation); - if (IsA(expr, Const) || IsA(expr, RelabelType)) { + if (eval_const && (IsA(expr, Const) || IsA(expr, RelabelType))) { result = eval_const_expression_value(NULL, result, NULL); } return result; diff --git a/src/common/backend/parser/parse_collate.cpp b/src/common/backend/parser/parse_collate.cpp index 291e0dec0..392c21969 100644 --- a/src/common/backend/parser/parse_collate.cpp +++ b/src/common/backend/parser/parse_collate.cpp @@ -626,6 +626,12 @@ static bool assign_collations_walker(Node* node, assign_collations_context* cont } if (IsA(node, Const)) { derivation = ((Const*)node)->constisnull ? DERIVATION_IGNORABLE : DERIVATION_COERCIBLE; + } else if (IsA(node, Param)) { + /* + * We set bind param DERIVATION_COERCIBLE to make "column = $1" and "column = 'string'" + * use the same collation to compare. + */ + derivation = (((Param*)node)->is_bind_param) ? DERIVATION_COERCIBLE : DERIVATION_IMPLICIT; } else { derivation = DERIVATION_IMPLICIT; } diff --git a/src/common/backend/parser/parse_param.cpp b/src/common/backend/parser/parse_param.cpp index 6cf7a288b..7f220ee27 100644 --- a/src/common/backend/parser/parse_param.cpp +++ b/src/common/backend/parser/parse_param.cpp @@ -108,9 +108,14 @@ static Node* fixed_paramref_hook(ParseState* pstate, ParamRef* pref) param->paramid = paramno; param->paramtype = parstate->paramTypes[paramno - 1]; param->paramtypmod = -1; - param->paramcollid = get_typcollation(param->paramtype); + if (OidIsValid(GetCollationConnection()) && IsSupportCharsetType(param->paramtype)) { + param->paramcollid = GetCollationConnection(); + } else { + param->paramcollid = get_typcollation(param->paramtype); + } param->location = pref->location; param->tableOfIndexTypeList = NULL; + param->is_bind_param = true; return (Node*)param; } @@ -171,6 +176,7 @@ static Node* variable_paramref_hook(ParseState* pstate, ParamRef* pref) } param->location = pref->location; param->tableOfIndexTypeList = NULL; + param->is_bind_param = true; return (Node*)param; } diff --git a/src/common/backend/utils/cache/plancache.cpp b/src/common/backend/utils/cache/plancache.cpp index d4de8f79c..efaca5e77 100644 --- a/src/common/backend/utils/cache/plancache.cpp +++ b/src/common/backend/utils/cache/plancache.cpp @@ -292,6 +292,7 @@ CachedPlanSource* CreateCachedPlan(Node* raw_parse_tree, const char* query_strin plansource->hasSubQuery = false; plansource->gpc_lockid = -1; plansource->hasSubQuery = false; + plansource->param_collation = GetCollationConnection(); #ifdef ENABLE_MOT @@ -378,6 +379,7 @@ CachedPlanSource* CreateOneShotCachedPlan(Node* raw_parse_tree, const char* quer plansource->total_custom_cost = 0; plansource->num_custom_plans = 0; plansource->spi_signature = {(uint32)-1, 0, (uint32)-1, -1}; + plansource->param_collation = GetCollationConnection(); #ifdef ENABLE_MOT plansource->storageEngineType = SE_TYPE_UNSPECIFIED; @@ -883,8 +885,11 @@ List* RevalidateCachedQuery(CachedPlanSource* plansource, bool has_lp) * If the query rewrite phase had a possible RLS dependency, we must redo * it if either the role setting has changed. */ - if (plansource->is_valid && plansource->dependsOnRole && (plansource->rewriteRoleId != GetUserId())) + if (plansource->is_valid && + ((plansource->dependsOnRole && (plansource->rewriteRoleId != GetUserId())) || + plansource->param_collation != GetCollationConnection())) { plansource->is_valid = false; + } /* * If the query is currently valid, acquire locks on the referenced @@ -1115,6 +1120,7 @@ List* RevalidateCachedQuery(CachedPlanSource* plansource, bool has_lp) plansource->query_context = querytree_context; plansource->query_list = qlist; + plansource->param_collation = GetCollationConnection(); /* Update ExecNodes for Light CN */ if (need_reset_singlenode || has_lp) { @@ -1214,7 +1220,9 @@ bool CheckCachedPlan(CachedPlanSource* plansource, CachedPlan *plan) Assert(!plan->is_oneshot); /* If plan isn't valid for current role, we can't use it. */ - if (plan->is_valid && plan->dependsOnRole && plan->planRoleId != GetUserId()) + if (plan->is_valid && + ((plan->dependsOnRole && plan->planRoleId != GetUserId()) || + plan->param_collation != GetCollationConnection())) plan->is_valid = false; /* @@ -1479,6 +1487,7 @@ CachedPlan* BuildCachedPlan(CachedPlanSource* plansource, List* qlist, ParamList */ plan->planRoleId = GetUserId(); plan->dependsOnRole = plansource->dependsOnRole; + plan->param_collation = GetCollationConnection(); foreach (lc, plist) { PlannedStmt* plannedstmt = (PlannedStmt*)lfirst(lc); @@ -2662,6 +2671,7 @@ CachedPlanSource* CopyCachedPlan(CachedPlanSource* plansource, bool is_share) newsource->spi_signature = plansource->spi_signature; newsource->gplan_is_fqs = plansource->gplan_is_fqs; newsource->nextval_default_expr_type = plansource->nextval_default_expr_type; + newsource->param_collation = plansource->param_collation; #ifdef ENABLE_MOT newsource->storageEngineType = SE_TYPE_UNSPECIFIED; diff --git a/src/common/backend/utils/init/globals.cpp b/src/common/backend/utils/init/globals.cpp index 139cb5b62..99f82fd13 100644 --- a/src/common/backend/utils/init/globals.cpp +++ b/src/common/backend/utils/init/globals.cpp @@ -75,12 +75,13 @@ bool will_shutdown = false; * NEXT | 92899 | ? | ? * ********************************************/ -const uint32 GRAND_VERSION_NUM = 92906; +const uint32 GRAND_VERSION_NUM = 92907; /******************************************** * 2.VERSION NUM FOR EACH FEATURE * Please write indescending order. ********************************************/ +const uint32 PARAM_MARK_VERSION_NUM = 92907; const uint32 TIMESCALE_DB_VERSION_NUM = 92904; const uint32 MULTI_CHARSET_VERSION_NUM = 92903; const uint32 NBTREE_INSERT_OPTIMIZATION_VERSION_NUM = 92902; diff --git a/src/common/backend/utils/mb/mbutils.cpp b/src/common/backend/utils/mb/mbutils.cpp index 6b7fcd3ff..e336cea1d 100644 --- a/src/common/backend/utils/mb/mbutils.cpp +++ b/src/common/backend/utils/mb/mbutils.cpp @@ -619,6 +619,43 @@ char* pg_client_to_server(const char* s, int len) return pg_any_to_server(s, len, u_sess->mb_cxt.ClientEncoding->encoding); } +char* verify_string_for_ascii(const char* s, int len, int encoding, bool bulkload_illegal_chars_conversion) +{ + /* + * No conversion is possible, but we must still validate the data, + * because the client-side code might have done string escaping using + * the selected client_encoding. If the client encoding is ASCII-safe + * then we just do a straight validation under that encoding. For an + * ASCII-unsafe encoding we have a problem: we dare not pass such data + * to the parser but we have no way to convert it. We compromise by + * rejecting the data if it contains any non-ASCII characters. + */ + if (PG_VALID_BE_ENCODING(encoding)) { + (void)pg_verify_mbstr(encoding, s, len, false); + return (char*)s; + } + + int i; + for (i = 0; i < len; i++) { + if (s[i] == '\0' || IS_HIGHBIT_SET(s[i])) { + if (!bulkload_illegal_chars_conversion) { + ereport(ERROR, + (errcode(ERRCODE_CHARACTER_NOT_IN_REPERTOIRE), + errmsg("invalid byte value for encoding \"%s\": 0x%02x", + pg_enc2name_tbl[PG_SQL_ASCII].name, + (unsigned char)s[i]))); + } + + if (s[i] == '\0') { + *((char*)&s[i]) = ' '; + } else { + *((char*)&s[i]) = '?'; + } + } + } + return (char*)s; +} + /* * convert any encoding to server encoding. */ @@ -645,38 +682,7 @@ char* pg_any_to_server(const char* s, int len, int encoding) } if (u_sess->mb_cxt.DatabaseEncoding->encoding == PG_SQL_ASCII) { - /* - * No conversion is possible, but we must still validate the data, - * because the client-side code might have done string escaping using - * the selected client_encoding. If the client encoding is ASCII-safe - * then we just do a straight validation under that encoding. For an - * ASCII-unsafe encoding we have a problem: we dare not pass such data - * to the parser but we have no way to convert it. We compromise by - * rejecting the data if it contains any non-ASCII characters. - */ - if (PG_VALID_BE_ENCODING(encoding)) { - (void)pg_verify_mbstr(encoding, s, len, false); - } else { - int i; - for (i = 0; i < len; i++) { - if (s[i] == '\0' || IS_HIGHBIT_SET(s[i])) { - if (bulkload_illegal_chars_conversion) { - if (s[i] == '\0') { - *((char*)&s[i]) = ' '; - } else { - *((char*)&s[i]) = '?'; - } - } else { - ereport(ERROR, - (errcode(ERRCODE_CHARACTER_NOT_IN_REPERTOIRE), - errmsg("invalid byte value for encoding \"%s\": 0x%02x", - pg_enc2name_tbl[PG_SQL_ASCII].name, - (unsigned char)s[i]))); - } - } - } - } - return (char*)s; + return verify_string_for_ascii(s, len, encoding, bulkload_illegal_chars_conversion); } if (u_sess->mb_cxt.ClientEncoding->encoding == encoding) { @@ -723,6 +729,44 @@ char* pg_any_to_client(const char* s, int len, int encoding, void* convert_finfo } } +/* + * convert client encoding to encoding. + */ +char* pg_client_to_any(const char* s, int len, int dst_encoding, void* convert_finfo) +{ + bool bulkload_illegal_chars_conversion = false; + + Assert(u_sess->mb_cxt.ClientEncoding); + + if (len <= 0) { + return (char*)s; + } + if (u_sess->cmd_cxt.bulkload_compatible_illegal_chars) { + bulkload_illegal_chars_conversion = true; + } + + int client_encoding = u_sess->mb_cxt.ClientEncoding->encoding; + if (client_encoding == dst_encoding || client_encoding == PG_SQL_ASCII) { + /* + * No conversion is needed, but we must still validate the data. + */ + (void)pg_verify_mbstr(dst_encoding, s, len, false); + return (char*)s; + } + + if (dst_encoding == PG_SQL_ASCII) { + return verify_string_for_ascii(s, len, client_encoding, bulkload_illegal_chars_conversion); + } + + if (u_sess->mb_cxt.DatabaseEncoding->encoding == dst_encoding) { + return perform_default_encoding_conversion(s, len, true); + } else if (convert_finfo != NULL) { + return try_fast_encoding_conversion( (char*)s, len, client_encoding, dst_encoding, convert_finfo); + } else { + return (char*)pg_do_encoding_conversion((unsigned char*)s, len, client_encoding, dst_encoding); + } +} + /* * convert server encoding to client encoding. */ diff --git a/src/gausskernel/optimizer/commands/prepare.cpp b/src/gausskernel/optimizer/commands/prepare.cpp index 967119f3c..cd0c556d7 100755 --- a/src/gausskernel/optimizer/commands/prepare.cpp +++ b/src/gausskernel/optimizer/commands/prepare.cpp @@ -494,6 +494,8 @@ static ParamListInfo EvaluateParams(CachedPlanSource* psrc, List* params, const ParamListInfo paramLI; List* exprstates = NIL; ListCell* l = NULL; + Oid param_collation; + int param_charset; int i; if (nparams != num_params) @@ -515,6 +517,8 @@ static ParamListInfo EvaluateParams(CachedPlanSource* psrc, List* params, const pstate = make_parsestate(NULL); pstate->p_sourcetext = queryString; + param_collation = GetCollationConnection(); + param_charset = GetCharsetConnection(); i = 0; foreach (l, params) { Node* expr = (Node*)lfirst(l); @@ -551,6 +555,12 @@ static ParamListInfo EvaluateParams(CachedPlanSource* psrc, List* params, const /* Take care of collations in the finished expression. */ assign_expr_collations(pstate, expr); + /* Try convert expression to target parameter charset. */ + if (OidIsValid(param_collation) && IsSupportCharsetType(expected_type_id)) { + /* convert charset only, expression will be evaluated below */ + expr = coerce_to_target_charset(expr, param_charset, expected_type_id, -1, param_collation, false); + } + lfirst(l) = expr; i++; } @@ -1860,6 +1870,7 @@ static Node* substitute_const_with_parameters_mutator(Node* node, substitute_con param->paramtypmod = con->consttypmod; param->paramcollid = con->constcollid; param->location = con->location; + param->is_bind_param = true; if (*context->args) { *context->args = (Oid*)repalloc(*context->args, param->paramid * sizeof(Oid)); } else { diff --git a/src/gausskernel/process/tcop/postgres.cpp b/src/gausskernel/process/tcop/postgres.cpp index 6fae4e501..b670a3a6a 100755 --- a/src/gausskernel/process/tcop/postgres.cpp +++ b/src/gausskernel/process/tcop/postgres.cpp @@ -3240,6 +3240,8 @@ static void exec_plan_with_params(StringInfo input_message) /* Get the parameter value */ if (numParams > 0) { int paramno; + Oid param_collation = GetCollationConnection(); + int param_charset = GetCharsetConnection(); params = (ParamListInfo)palloc(offsetof(ParamListInfoData, params) + numParams * sizeof(ParamExternData)); @@ -3312,6 +3314,8 @@ static void exec_plan_with_params(StringInfo input_message) */ if (isNull) { pstring = NULL; + } else if (OidIsValid(param_collation) && IsSupportCharsetType(ptype)) { + pstring = pg_client_to_any(pbuf.data, plength, param_charset); } else { pstring = pg_client_to_server(pbuf.data, plength); } @@ -4032,6 +4036,8 @@ static int getSingleNodeIdx(StringInfo input_message, CachedPlanSource* psrc, co */ if (numParams > 0) { int paramno; + Oid param_collation = GetCollationConnection(); + int param_charset = GetCharsetConnection(); params = (ParamListInfo)palloc(offsetof(ParamListInfoData, params) + numParams * sizeof(ParamExternData)); @@ -4111,10 +4117,13 @@ static int getSingleNodeIdx(StringInfo input_message, CachedPlanSource* psrc, co * We have to do encoding conversion before calling the * typinput routine. */ - if (isNull) + if (isNull) { pstring = NULL; - else + } else if (OidIsValid(param_collation) && IsSupportCharsetType(ptype)) { + pstring = pg_client_to_any(pbuf.data, plength, param_charset); + } else { pstring = pg_client_to_server(pbuf.data, plength); + } pval = OidInputFunctionCall(typinput, pstring, typioparam, -1); @@ -4339,6 +4348,8 @@ void exec_get_ddl_params(StringInfo input_message) if (numParams > 0) { int paramno; + Oid param_collation = GetCollationConnection(); + int param_charset = GetCharsetConnection(); params = (ParamListInfo)palloc(offsetof(ParamListInfoData, params) + numParams * sizeof(ParamExternData)); params->paramFetch = NULL; params->paramFetchArg = NULL; @@ -4392,7 +4403,14 @@ void exec_get_ddl_params(StringInfo input_message) getTypeInputInfo(ptype, &typinput, &typioparam); - pstring = isNull ? NULL : pg_client_to_server(pbuf.data, plength); + if (isNull) { + pstring = NULL; + } else if (OidIsValid(param_collation) && IsSupportCharsetType(ptype)) { + pstring = pg_client_to_any(pbuf.data, plength, param_charset); + } else { + pstring = pg_client_to_server(pbuf.data, plength); + } + pval = OidInputFunctionCall(typinput, pstring, typioparam, -1); /* Free result of encoding conversion, if any */ @@ -4822,6 +4840,8 @@ static void exec_bind_message(StringInfo input_message) */ if (numParams > 0) { int paramno; + Oid param_collation = GetCollationConnection(); + int param_charset = GetCharsetConnection(); params = (ParamListInfo)palloc(offsetof(ParamListInfoData, params) + numParams * sizeof(ParamExternData)); /* we have static list of params, so no hooks needed */ @@ -4909,10 +4929,13 @@ static void exec_bind_message(StringInfo input_message) * We have to do encoding conversion before calling the * typinput routine. */ - if (isNull) + if (isNull) { pstring = NULL; - else + } else if (OidIsValid(param_collation) && IsSupportCharsetType(ptype)) { + pstring = pg_client_to_any(pbuf.data, plength, param_charset); + } else { pstring = pg_client_to_server(pbuf.data, plength); + } #ifndef ENABLE_MULTIPLE_NODES if (pmode == NULL || *pmode != PROARGMODE_OUT || !enable_out_param_override()) { @@ -11314,6 +11337,9 @@ static void exec_batch_bind_execute(StringInfo input_message) bool save_log_statement_stats = u_sess->attr.attr_common.log_statement_stats; bool snapshot_set = false; char msec_str[PRINTF_DST_MAX]; + Oid param_collation = GetCollationConnection(); + int param_charset = GetCharsetConnection(); + FmgrInfo convert_finfo; int msg_type; /* D message */ @@ -11559,6 +11585,17 @@ static void exec_batch_bind_execute(StringInfo input_message) params_set_end[0] = input_message->cursor; } + /* + * There is a fast path for transcoding from the client to the server. + * If the characterset_connection is different from the server_encoding, + * Fmgrinfo should be constructed here to avoid poor construction performance during each conversion. + */ + if (OidIsValid(param_collation) && param_charset != GetDatabaseEncoding()) { + construct_conversion_fmgr_info(pg_get_client_encoding(), param_charset, (void*)&convert_finfo); + } else { + convert_finfo.fn_oid = InvalidOid; + } + /* Second, process each set of params */ params_set = (ParamListInfo*)palloc0(batch_count * sizeof(ParamListInfo)); if (numParams > 0) { @@ -11644,10 +11681,13 @@ static void exec_batch_bind_execute(StringInfo input_message) * We have to do encoding conversion before calling the * typinput routine. */ - if (isNull) + if (isNull) { pstring = NULL; - else + } else if (OidIsValid(param_collation) && IsSupportCharsetType(ptype)) { + pstring = pg_client_to_any(pbuf.data, plength, param_charset, (void*)&convert_finfo); + } else { pstring = pg_client_to_server(pbuf.data, plength); + } pval = OidInputFunctionCall(typinput, pstring, typioparam, -1); diff --git a/src/gausskernel/runtime/opfusion/opfusion.cpp b/src/gausskernel/runtime/opfusion/opfusion.cpp index 5c1826385..4cd2c4fe5 100644 --- a/src/gausskernel/runtime/opfusion/opfusion.cpp +++ b/src/gausskernel/runtime/opfusion/opfusion.cpp @@ -805,6 +805,8 @@ void OpFusion::updatePreAllocParamter(StringInfo input_message) } (void)MemoryContextSwitchTo(m_local.m_tmpContext); if (num_params > 0) { + Oid param_collation = GetCollationConnection(); + int param_charset = GetCharsetConnection(); for (paramno = 0; paramno < num_params; paramno++) { Oid ptype = m_global->m_psrc->param_types[paramno]; int32 plength; @@ -875,6 +877,8 @@ void OpFusion::updatePreAllocParamter(StringInfo input_message) */ if (isNull) { pstring = NULL; + } else if (OidIsValid(param_collation) && IsSupportCharsetType(ptype)) { + pstring = pg_client_to_any(pbuf.data, plength, param_charset); } else { pstring = pg_client_to_server(pbuf.data, plength); } diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h index fe36248c5..dadc6b8f9 100644 --- a/src/include/mb/pg_wchar.h +++ b/src/include/mb/pg_wchar.h @@ -490,6 +490,7 @@ extern char* pg_server_to_client(const char* s, int len); extern char* pg_any_to_server(const char* s, int len, int encoding); extern char* pg_server_to_any(const char* s, int len, int encoding, void* convert_finfo = NULL); extern char* pg_any_to_client(const char* s, int len, int encoding, void* convert_finfo = NULL); +extern char* pg_client_to_any(const char* s, int len, int dst_encoding, void* convert_finfo = NULL); extern bool WillTranscodingBePerformed(int encoding); extern unsigned short BIG5toCNS(unsigned short big5, unsigned char* lc); diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h index dfee1d6db..056ebc32a 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h @@ -38,6 +38,7 @@ * Backend version and inplace upgrade staffs *****************************************************************************/ extern const uint32 TXNSTATUS_CACHE_DFX_VERSION_NUM; +extern const uint32 PARAM_MARK_VERSION_NUM; extern const uint32 TIMESCALE_DB_VERSION_NUM; extern const uint32 NBTREE_INSERT_OPTIMIZATION_VERSION_NUM; extern const uint32 NBTREE_DEDUPLICATION_VERSION_NUM; diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h index f003da36d..7cfb51f0a 100644 --- a/src/include/nodes/primnodes.h +++ b/src/include/nodes/primnodes.h @@ -256,6 +256,7 @@ typedef struct Param { Oid tableOfIndexType; /* type Oid of table of (wait to discard) */ Oid recordVarTypOid; /* package record var's composite type oid */ List* tableOfIndexTypeList; /* type Oid list of table of, max size 6 */ + bool is_bind_param; } Param; /* diff --git a/src/include/parser/parse_coerce.h b/src/include/parser/parse_coerce.h index 1d54ae299..1fbd34b37 100644 --- a/src/include/parser/parse_coerce.h +++ b/src/include/parser/parse_coerce.h @@ -77,8 +77,8 @@ extern CoercionPathType find_coercion_pathway( extern CoercionPathType find_typmod_coercion_function(Oid typeId, Oid* funcid); extern void expression_error_callback(void* arg); -extern Node* coerce_to_target_charset( - Node* expr, int target_charset, Oid target_type, int32 target_typmod, Oid target_collation); +extern Node* coerce_to_target_charset(Node* expr, int target_charset, Oid target_type, int32 target_typmod, Oid target_collation, + bool eval_const = true); extern Node *transferConstToAconst(Node *node); diff --git a/src/include/utils/plancache.h b/src/include/utils/plancache.h index 6e5017e24..f0e6a36dc 100644 --- a/src/include/utils/plancache.h +++ b/src/include/utils/plancache.h @@ -422,6 +422,7 @@ typedef struct CachedPlanSource { bool opteval; bool hasSubQuery; int nextval_default_expr_type; + Oid param_collation; } CachedPlanSource; /* @@ -467,6 +468,7 @@ typedef struct CachedPlan { CachedPlanInfo *cpi; bool is_candidate; double cost; /* cost of generic plan, or -1 if not known */ + Oid param_collation; } CachedPlan; typedef struct CachedPlanInfo { diff --git a/src/test/regress/expected/charset_gbk_b_db.out b/src/test/regress/expected/charset_gbk_b_db.out index 6fddbd39c..06cb4915b 100644 --- a/src/test/regress/expected/charset_gbk_b_db.out +++ b/src/test/regress/expected/charset_gbk_b_db.out @@ -1922,60 +1922,157 @@ SELECT CONCAT(@var_binary, fgbk_bin) result, collation for(result) FROM t_diff_c (1 row) -- -- concat column and bind parameter +-- -- bind parameter collation is fixed as collation_connection, collation level is same as a const -- -- -- -- PBE with implicit collation PREPARE test_merge_collation(text) AS SELECT CONCAT(futf8_uni, $1) result, collation for(result) FROM t_diff_charset_columns; -EXECUTE test_merge_collation(_utf8mb4'高斯DB'); -- $1 use collation_connection, futf8_uni +-- -- -- -- -- _utf8mb4 +SET @pbe_param1 = _utf8mb4'高斯DB'; +EXECUTE test_merge_collation(@pbe_param1); -- futf8_uni collation has priority + result | pg_collation_for +----------------+-------------------- + 高斯db˹DB | utf8mb4_unicode_ci +(1 row) + +EXECUTE test_merge_collation(_utf8mb4'高斯DB'); -- same as above + result | pg_collation_for +----------------+-------------------- + 高斯db˹DB | utf8mb4_unicode_ci +(1 row) + +SELECT CONCAT(futf8_uni, _utf8mb4'高斯DB') result, collation for(result) FROM t_diff_charset_columns; -- same as above + result | pg_collation_for +----------------+-------------------- + 高斯db˹DB | utf8mb4_unicode_ci +(1 row) + +-- -- -- -- -- _gbk +SET @pbe_param1 = _gbk'高斯DB'; +EXECUTE test_merge_collation(@pbe_param1); -- _gbk noneffective, futf8_uni collation has priority, _gbk'高斯DB' will not convert to utf8mb4 result | pg_collation_for ------------------+-------------------- 高斯db高斯DB | utf8mb4_unicode_ci (1 row) -EXECUTE test_merge_collation(_utf8mb4'高斯DB' collate utf8mb4_unicode_ci); -- explicit noneffective, futf8_uni +EXECUTE test_merge_collation(_gbk'高斯DB'); -- same as above result | pg_collation_for ------------------+-------------------- 高斯db高斯DB | utf8mb4_unicode_ci (1 row) -EXECUTE test_merge_collation(_gbk'高斯DB'); -- _gbk noneffective, futf8_uni +SELECT CONCAT(futf8_uni, _gbk'高斯DB') result, collation for(result) FROM t_diff_charset_columns; -- same as above result | pg_collation_for ------------------+-------------------- 高斯db高斯DB | utf8mb4_unicode_ci (1 row) +-- -- -- -- -- _utf8mb4 utf8mb4_unicode_ci +SET @pbe_param1 = _utf8mb4'高斯DB' collate utf8mb4_bin; +EXECUTE test_merge_collation(@pbe_param1); -- explicit noneffective, futf8_uni collation has priority + result | pg_collation_for +----------------+-------------------- + 高斯db˹DB | utf8mb4_unicode_ci +(1 row) + +EXECUTE test_merge_collation(_utf8mb4'高斯DB' collate utf8mb4_unicode_ci); -- explicit noneffective, futf8_uni collation has priority + result | pg_collation_for +----------------+-------------------- + 高斯db˹DB | utf8mb4_unicode_ci +(1 row) + DEALLOCATE test_merge_collation; --- -- -- -- PBE with implicit collation -PREPARE test_merge_collation(text) AS -SELECT CONCAT($1, fgb18030_bin) result, collation for(result) FROM t_diff_charset_columns; -- $1 use collation_connection, ERROR -ERROR: collation mismatch between collations "gbk_chinese_ci" and "gb18030_bin" -LINE 2: ...) result, collation for(result) FROM t_diff_charset_columns; - ^ -DEALLOCATE test_merge_collation; -ERROR: prepared statement "test_merge_collation" does not exist --- -- -- -- PBE with explicit collation, -PREPARE test_merge_collation(text) AS -SELECT CONCAT($1 collate utf8mb4_unicode_ci, futf8_bin) result, collation for(result) FROM t_diff_charset_columns; -- $1 use collation_connection, ERROR -ERROR: COLLATION "utf8mb4_unicode_ci" is not valid for CHARACTER SET "GBK" -LINE 2: SELECT CONCAT($1 collate utf8mb4_unicode_ci, futf8_bin) resu... - ^ -DEALLOCATE test_merge_collation; -ERROR: prepared statement "test_merge_collation" does not exist -- -- -- -- PBE with explicit collation, PREPARE test_merge_collation(text) AS SELECT CONCAT($1 collate gbk_chinese_ci, futf8_bin) result, collation for(result) FROM t_diff_charset_columns; -EXECUTE test_merge_collation(_utf8mb4'高斯DB'); -- gbk_chinese_ci +-- -- -- -- -- _utf8mb4 +SET @pbe_param1 = _utf8mb4'高斯DB'; +EXECUTE test_merge_collation(@pbe_param1); + result | pg_collation_for +----------------+------------------ + ˹DB高斯DB | gbk_chinese_ci +(1 row) + +EXECUTE test_merge_collation(_utf8mb4'高斯DB'); -- utf8mb4_unicode_ci + result | pg_collation_for +----------------+------------------ + ˹DB高斯DB | gbk_chinese_ci +(1 row) + +-- -- -- -- -- _gbk +SET @pbe_param1 = _gbk'高斯DB'; +EXECUTE test_merge_collation(@pbe_param1); result | pg_collation_for ------------------+------------------ 高斯DB高斯DB | gbk_chinese_ci (1 row) -EXECUTE test_merge_collation(_gbk'高斯DB'); -- gbk_chinese_ci +EXECUTE test_merge_collation(_gbk'高斯DB'); -- utf8mb4_unicode_ci result | pg_collation_for ------------------+------------------ 高斯DB高斯DB | gbk_chinese_ci (1 row) DEALLOCATE test_merge_collation; +-- -- -- -- PBE with explicit collation, +PREPARE test_merge_collation(text) AS +SELECT CONCAT($1 collate utf8mb4_unicode_ci, futf8_bin) result, collation for(result) FROM t_diff_charset_columns; -- $1 use collation_connection, ERROR +ERROR: COLLATION "utf8mb4_unicode_ci" is not valid for CHARACTER SET "GBK" +LINE 2: SELECT CONCAT($1 collate utf8mb4_unicode_ci, futf8_bin) resu... + ^ +DEALLOCATE test_merge_collation; +ERROR: prepared statement "test_merge_collation" does not exist +-- -- -- -- test revalidate +SELECT fgbk_chi result FROM t_diff_charset_columns WHERE fgbk_chi=_gbk'高斯db'; -- 1 rows + result +---------- + 高斯db +(1 row) + +PREPARE test_revalidate(text) AS +SELECT fgbk_chi result FROM t_diff_charset_columns WHERE fgbk_chi=$1; +EXECUTE test_revalidate(_gbk'高斯db'); -- fgbk_chi collation has priority, 1 rows + result +---------- + 高斯db +(1 row) + +ALTER INDEX idx_prefixkey_futf8_bin UNUSABLE; +EXECUTE test_revalidate(_gbk'高斯db'); -- fgbk_chi collation has priority, 1 rows + result +---------- + 高斯db +(1 row) + +SET NAMES utf8mb4 COLLATE utf8mb4_bin; +EXECUTE test_revalidate(_gbk'高斯db'); -- fgbk_chi collation has priority, 1 rows + result +---------- + 楂樻柉db +(1 row) + +ALTER INDEX idx_prefixkey_futf8_bin REBUILD; +EXECUTE test_revalidate(_gbk'高斯db'); -- fgbk_chi collation has priority, 1 rows + result +---------- + 楂樻柉db +(1 row) + +SET NAMES gbk COLLATE gbk_bin; +EXECUTE test_revalidate(_gbk'高斯db'); -- fgbk_chi collation has priority, 1 rows + result +---------- + 高斯db +(1 row) + +ALTER INDEX idx_prefixkey_futf8_bin REBUILD; +EXECUTE test_revalidate(_gbk'高斯db'); -- fgbk_chi collation has priority, 1 rows + result +---------- + 高斯db +(1 row) + +DEALLOCATE test_revalidate; +SET NAMES gbk; -- -- concat for DERIVATION -- -- -- same charset & diff DERIVATION SELECT CONCAT(CONCAT(futf8_gen, futf8_uni), futf8_bin) result, collation for(result) FROM t_diff_charset_columns; -- conflict in concat inside diff --git a/src/test/regress/expected/charset_utf8mb4_b_db.out b/src/test/regress/expected/charset_utf8mb4_b_db.out index 5fead0c32..d120955a3 100644 --- a/src/test/regress/expected/charset_utf8mb4_b_db.out +++ b/src/test/regress/expected/charset_utf8mb4_b_db.out @@ -2313,63 +2313,96 @@ SELECT CONCAT(@var_binary, fgbk_bin) result, collation for(result) FROM t_diff_c (1 row) -- -- concat column and bind parameter +-- -- bind parameter collation is fixed as collation_connection, collation level is same as a const -- -- -- -- PBE with implicit collation PREPARE test_merge_collation(text) AS SELECT CONCAT(futf8_uni, $1) result, collation for(result) FROM t_diff_charset_columns; -EXECUTE test_merge_collation(_utf8mb4'高斯DB'); -- $1 use collation_connection, conflict - result | pg_collation_for ---------------+------------------ - 高斯db高斯DB | +-- -- -- -- -- _utf8mb4 +SET @pbe_param1 = _utf8mb4'高斯DB'; +EXECUTE test_merge_collation(@pbe_param1); -- futf8_uni collation has priority + result | pg_collation_for +--------------+-------------------- + 高斯db高斯DB | utf8mb4_unicode_ci (1 row) -EXECUTE test_merge_collation(_utf8mb4'高斯DB' collate utf8mb4_unicode_ci); -- explicit noneffective, conflict - result | pg_collation_for ---------------+------------------ - 高斯db高斯DB | +EXECUTE test_merge_collation(_utf8mb4'高斯DB'); -- same as above + result | pg_collation_for +--------------+-------------------- + 高斯db高斯DB | utf8mb4_unicode_ci (1 row) -EXECUTE test_merge_collation(_gbk'高斯DB'); -- _gbk noneffective, conflict - result | pg_collation_for ---------------+------------------ - 高斯db高斯DB | +SELECT CONCAT(futf8_uni, _utf8mb4'高斯DB') result, collation for(result) FROM t_diff_charset_columns; -- same as above + result | pg_collation_for +--------------+-------------------- + 高斯db高斯DB | utf8mb4_unicode_ci (1 row) -DEALLOCATE test_merge_collation; --- -- -- -- PBE with implicit collation -PREPARE test_merge_collation(text) AS -SELECT CONCAT($1, fgbk_bin) result, collation for(result) FROM t_diff_charset_columns; -EXECUTE test_merge_collation(_utf8mb4'高斯DB'); -- $1 use collation_connection, utf8_gen +-- -- -- -- -- _gbk +SET @pbe_param1 = _gbk'高斯DB'; +EXECUTE test_merge_collation(@pbe_param1); -- _gbk noneffective, futf8_uni collation has priority, _gbk'高斯DB' will not convert to utf8mb4 + result | pg_collation_for +----------------+-------------------- + 高斯db楂樻柉DB | utf8mb4_unicode_ci +(1 row) + +EXECUTE test_merge_collation(_gbk'高斯DB'); -- same as above + result | pg_collation_for +----------------+-------------------- + 高斯db楂樻柉DB | utf8mb4_unicode_ci +(1 row) + +SELECT CONCAT(futf8_uni, _gbk'高斯DB') result, collation for(result) FROM t_diff_charset_columns; -- same as above + result | pg_collation_for +----------------+-------------------- + 高斯db楂樻柉DB | utf8mb4_unicode_ci +(1 row) + +-- -- -- -- -- _utf8mb4 utf8mb4_unicode_ci +SET @pbe_param1 = _utf8mb4'高斯DB' collate utf8mb4_bin; +EXECUTE test_merge_collation(@pbe_param1); -- explicit noneffective, futf8_uni collation has priority result | pg_collation_for --------------+-------------------- - 高斯DB高斯DB | utf8mb4_general_ci + 高斯db高斯DB | utf8mb4_unicode_ci (1 row) -EXECUTE test_merge_collation(_gbk'高斯DB'); -- _gbk noneffective, utf8_gen +EXECUTE test_merge_collation(_utf8mb4'高斯DB' collate utf8mb4_unicode_ci); -- explicit noneffective, futf8_uni collation has priority result | pg_collation_for --------------+-------------------- - 高斯DB高斯DB | utf8mb4_general_ci + 高斯db高斯DB | utf8mb4_unicode_ci (1 row) -EXECUTE test_merge_collation(_utf8mb4'高斯DB' collate gbk_chinese_ci); -- implicit type cast, keep explicit collation and check it, ERROR -ERROR: COLLATION "gbk_chinese_ci" is not valid for CHARACTER SET "UTF8" -LINE 1: EXECUTE test_merge_collation(_utf8mb4'高斯DB' collate gbk_ch... - ^ DEALLOCATE test_merge_collation; -- -- -- -- PBE with explicit collation, PREPARE test_merge_collation(text) AS SELECT CONCAT($1 collate utf8mb4_unicode_ci, futf8_bin) result, collation for(result) FROM t_diff_charset_columns; -EXECUTE test_merge_collation(_utf8mb4'高斯DB'); -- utf8mb4_unicode_ci +-- -- -- -- -- _utf8mb4 +SET @pbe_param1 = _utf8mb4'高斯DB'; +EXECUTE test_merge_collation(@pbe_param1); result | pg_collation_for --------------+-------------------- 高斯DB高斯DB | utf8mb4_unicode_ci (1 row) -EXECUTE test_merge_collation(_gbk'高斯DB'); -- utf8mb4_unicode_ci +EXECUTE test_merge_collation(_utf8mb4'高斯DB'); -- utf8mb4_unicode_ci result | pg_collation_for --------------+-------------------- 高斯DB高斯DB | utf8mb4_unicode_ci (1 row) +-- -- -- -- -- _gbk +SET @pbe_param1 = _gbk'高斯DB'; +EXECUTE test_merge_collation(@pbe_param1); + result | pg_collation_for +----------------+-------------------- + 楂樻柉DB高斯DB | utf8mb4_unicode_ci +(1 row) + +EXECUTE test_merge_collation(_gbk'高斯DB'); -- utf8mb4_unicode_ci + result | pg_collation_for +----------------+-------------------- + 楂樻柉DB高斯DB | utf8mb4_unicode_ci +(1 row) + DEALLOCATE test_merge_collation; -- -- -- -- PBE with explicit collation, PREPARE test_merge_collation(text) AS @@ -2377,8 +2410,58 @@ SELECT CONCAT($1 collate gbk_chinese_ci, futf8_bin) result, collation for(result ERROR: COLLATION "gbk_chinese_ci" is not valid for CHARACTER SET "UTF8" LINE 2: SELECT CONCAT($1 collate gbk_chinese_ci, futf8_bin) result, ... ^ -DEALLOCATE test_merge_collation; -ERROR: prepared statement "test_merge_collation" does not exist +-- -- -- -- test revalidate +SELECT fgbk_chi result FROM t_diff_charset_columns WHERE fgbk_chi=_utf8mb4'高斯db'; -- 1 rows + result +-------- + 高斯db +(1 row) + +PREPARE test_revalidate(text) AS +SELECT fgbk_chi result FROM t_diff_charset_columns WHERE fgbk_chi=$1; +EXECUTE test_revalidate(_utf8mb4'高斯db'); -- fgbk_chi collation has priority, 1 rows + result +-------- + 高斯db +(1 row) + +ALTER INDEX idx_prefixkey_futf8_bin UNUSABLE; +EXECUTE test_revalidate(_utf8mb4'高斯db'); -- fgbk_chi collation has priority, 1 rows + result +-------- + 高斯db +(1 row) + +SET NAMES utf8mb4 COLLATE utf8mb4_bin; +EXECUTE test_revalidate(_utf8mb4'高斯db'); -- fgbk_chi collation has priority, 1 rows + result +-------- + 高斯db +(1 row) + +ALTER INDEX idx_prefixkey_futf8_bin REBUILD; +EXECUTE test_revalidate(_utf8mb4'高斯db'); -- fgbk_chi collation has priority, 1 rows + result +-------- + 高斯db +(1 row) + +SET NAMES gbk COLLATE gbk_bin; +EXECUTE test_revalidate(_utf8mb4'高斯db'); -- fgbk_chi collation has priority, 1 rows + result +-------- + ˹db +(1 row) + +ALTER INDEX idx_prefixkey_futf8_bin REBUILD; +EXECUTE test_revalidate(_utf8mb4'高斯db'); -- fgbk_chi collation has priority, 1 rows + result +-------- + ˹db +(1 row) + +DEALLOCATE test_revalidate; +SET NAMES utf8mb4; -- -- concat column and PROCEDURE parameter with CURSOR -- -- -- implicit collation && string create or replace procedure merge_collation_func(p1 text, p2 text) diff --git a/src/test/regress/sql/charset_gbk_b_db.sql b/src/test/regress/sql/charset_gbk_b_db.sql index d6184389e..501fed2f4 100644 --- a/src/test/regress/sql/charset_gbk_b_db.sql +++ b/src/test/regress/sql/charset_gbk_b_db.sql @@ -440,27 +440,59 @@ SELECT CONCAT(futf8_bin, @var_binary) result, collation for(result) FROM t_diff_ SELECT CONCAT(@var_binary, fgbk_bin) result, collation for(result) FROM t_diff_charset_columns; -- -- concat column and bind parameter +-- -- bind parameter collation is fixed as collation_connection, collation level is same as a const -- -- -- -- PBE with implicit collation PREPARE test_merge_collation(text) AS SELECT CONCAT(futf8_uni, $1) result, collation for(result) FROM t_diff_charset_columns; -EXECUTE test_merge_collation(_utf8mb4'高斯DB'); -- $1 use collation_connection, futf8_uni -EXECUTE test_merge_collation(_utf8mb4'高斯DB' collate utf8mb4_unicode_ci); -- explicit noneffective, futf8_uni -EXECUTE test_merge_collation(_gbk'高斯DB'); -- _gbk noneffective, futf8_uni -DEALLOCATE test_merge_collation; --- -- -- -- PBE with implicit collation -PREPARE test_merge_collation(text) AS -SELECT CONCAT($1, fgb18030_bin) result, collation for(result) FROM t_diff_charset_columns; -- $1 use collation_connection, ERROR +-- -- -- -- -- _utf8mb4 +SET @pbe_param1 = _utf8mb4'高斯DB'; +EXECUTE test_merge_collation(@pbe_param1); -- futf8_uni collation has priority +EXECUTE test_merge_collation(_utf8mb4'高斯DB'); -- same as above +SELECT CONCAT(futf8_uni, _utf8mb4'高斯DB') result, collation for(result) FROM t_diff_charset_columns; -- same as above +-- -- -- -- -- _gbk +SET @pbe_param1 = _gbk'高斯DB'; +EXECUTE test_merge_collation(@pbe_param1); -- _gbk noneffective, futf8_uni collation has priority, _gbk'高斯DB' will not convert to utf8mb4 +EXECUTE test_merge_collation(_gbk'高斯DB'); -- same as above +SELECT CONCAT(futf8_uni, _gbk'高斯DB') result, collation for(result) FROM t_diff_charset_columns; -- same as above +-- -- -- -- -- _utf8mb4 utf8mb4_unicode_ci +SET @pbe_param1 = _utf8mb4'高斯DB' collate utf8mb4_bin; +EXECUTE test_merge_collation(@pbe_param1); -- explicit noneffective, futf8_uni collation has priority +EXECUTE test_merge_collation(_utf8mb4'高斯DB' collate utf8mb4_unicode_ci); -- explicit noneffective, futf8_uni collation has priority DEALLOCATE test_merge_collation; -- -- -- -- PBE with explicit collation, PREPARE test_merge_collation(text) AS -SELECT CONCAT($1 collate utf8mb4_unicode_ci, futf8_bin) result, collation for(result) FROM t_diff_charset_columns; -- $1 use collation_connection, ERROR +SELECT CONCAT($1 collate gbk_chinese_ci, futf8_bin) result, collation for(result) FROM t_diff_charset_columns; +-- -- -- -- -- _utf8mb4 +SET @pbe_param1 = _utf8mb4'高斯DB'; +EXECUTE test_merge_collation(@pbe_param1); +EXECUTE test_merge_collation(_utf8mb4'高斯DB'); -- utf8mb4_unicode_ci +-- -- -- -- -- _gbk +SET @pbe_param1 = _gbk'高斯DB'; +EXECUTE test_merge_collation(@pbe_param1); +EXECUTE test_merge_collation(_gbk'高斯DB'); -- utf8mb4_unicode_ci DEALLOCATE test_merge_collation; -- -- -- -- PBE with explicit collation, PREPARE test_merge_collation(text) AS -SELECT CONCAT($1 collate gbk_chinese_ci, futf8_bin) result, collation for(result) FROM t_diff_charset_columns; -EXECUTE test_merge_collation(_utf8mb4'高斯DB'); -- gbk_chinese_ci -EXECUTE test_merge_collation(_gbk'高斯DB'); -- gbk_chinese_ci +SELECT CONCAT($1 collate utf8mb4_unicode_ci, futf8_bin) result, collation for(result) FROM t_diff_charset_columns; -- $1 use collation_connection, ERROR DEALLOCATE test_merge_collation; +-- -- -- -- test revalidate +SELECT fgbk_chi result FROM t_diff_charset_columns WHERE fgbk_chi=_gbk'高斯db'; -- 1 rows +PREPARE test_revalidate(text) AS +SELECT fgbk_chi result FROM t_diff_charset_columns WHERE fgbk_chi=$1; +EXECUTE test_revalidate(_gbk'高斯db'); -- fgbk_chi collation has priority, 1 rows +ALTER INDEX idx_prefixkey_futf8_bin UNUSABLE; +EXECUTE test_revalidate(_gbk'高斯db'); -- fgbk_chi collation has priority, 1 rows +SET NAMES utf8mb4 COLLATE utf8mb4_bin; +EXECUTE test_revalidate(_gbk'高斯db'); -- fgbk_chi collation has priority, 1 rows +ALTER INDEX idx_prefixkey_futf8_bin REBUILD; +EXECUTE test_revalidate(_gbk'高斯db'); -- fgbk_chi collation has priority, 1 rows +SET NAMES gbk COLLATE gbk_bin; +EXECUTE test_revalidate(_gbk'高斯db'); -- fgbk_chi collation has priority, 1 rows +ALTER INDEX idx_prefixkey_futf8_bin REBUILD; +EXECUTE test_revalidate(_gbk'高斯db'); -- fgbk_chi collation has priority, 1 rows +DEALLOCATE test_revalidate; +SET NAMES gbk; + -- -- concat for DERIVATION -- -- -- same charset & diff DERIVATION diff --git a/src/test/regress/sql/charset_utf8mb4_b_db.sql b/src/test/regress/sql/charset_utf8mb4_b_db.sql index a6ab68d58..af0169dda 100644 --- a/src/test/regress/sql/charset_utf8mb4_b_db.sql +++ b/src/test/regress/sql/charset_utf8mb4_b_db.sql @@ -569,30 +569,57 @@ SELECT CONCAT(futf8_bin, @var_binary) result, collation for(result) FROM t_diff_ SELECT CONCAT(@var_binary, fgbk_bin) result, collation for(result) FROM t_diff_charset_columns; -- -- concat column and bind parameter +-- -- bind parameter collation is fixed as collation_connection, collation level is same as a const -- -- -- -- PBE with implicit collation PREPARE test_merge_collation(text) AS SELECT CONCAT(futf8_uni, $1) result, collation for(result) FROM t_diff_charset_columns; -EXECUTE test_merge_collation(_utf8mb4'高斯DB'); -- $1 use collation_connection, conflict -EXECUTE test_merge_collation(_utf8mb4'高斯DB' collate utf8mb4_unicode_ci); -- explicit noneffective, conflict -EXECUTE test_merge_collation(_gbk'高斯DB'); -- _gbk noneffective, conflict -DEALLOCATE test_merge_collation; --- -- -- -- PBE with implicit collation -PREPARE test_merge_collation(text) AS -SELECT CONCAT($1, fgbk_bin) result, collation for(result) FROM t_diff_charset_columns; -EXECUTE test_merge_collation(_utf8mb4'高斯DB'); -- $1 use collation_connection, utf8_gen -EXECUTE test_merge_collation(_gbk'高斯DB'); -- _gbk noneffective, utf8_gen -EXECUTE test_merge_collation(_utf8mb4'高斯DB' collate gbk_chinese_ci); -- implicit type cast, keep explicit collation and check it, ERROR +-- -- -- -- -- _utf8mb4 +SET @pbe_param1 = _utf8mb4'高斯DB'; +EXECUTE test_merge_collation(@pbe_param1); -- futf8_uni collation has priority +EXECUTE test_merge_collation(_utf8mb4'高斯DB'); -- same as above +SELECT CONCAT(futf8_uni, _utf8mb4'高斯DB') result, collation for(result) FROM t_diff_charset_columns; -- same as above +-- -- -- -- -- _gbk +SET @pbe_param1 = _gbk'高斯DB'; +EXECUTE test_merge_collation(@pbe_param1); -- _gbk noneffective, futf8_uni collation has priority, _gbk'高斯DB' will not convert to utf8mb4 +EXECUTE test_merge_collation(_gbk'高斯DB'); -- same as above +SELECT CONCAT(futf8_uni, _gbk'高斯DB') result, collation for(result) FROM t_diff_charset_columns; -- same as above +-- -- -- -- -- _utf8mb4 utf8mb4_unicode_ci +SET @pbe_param1 = _utf8mb4'高斯DB' collate utf8mb4_bin; +EXECUTE test_merge_collation(@pbe_param1); -- explicit noneffective, futf8_uni collation has priority +EXECUTE test_merge_collation(_utf8mb4'高斯DB' collate utf8mb4_unicode_ci); -- explicit noneffective, futf8_uni collation has priority DEALLOCATE test_merge_collation; -- -- -- -- PBE with explicit collation, PREPARE test_merge_collation(text) AS SELECT CONCAT($1 collate utf8mb4_unicode_ci, futf8_bin) result, collation for(result) FROM t_diff_charset_columns; +-- -- -- -- -- _utf8mb4 +SET @pbe_param1 = _utf8mb4'高斯DB'; +EXECUTE test_merge_collation(@pbe_param1); EXECUTE test_merge_collation(_utf8mb4'高斯DB'); -- utf8mb4_unicode_ci +-- -- -- -- -- _gbk +SET @pbe_param1 = _gbk'高斯DB'; +EXECUTE test_merge_collation(@pbe_param1); EXECUTE test_merge_collation(_gbk'高斯DB'); -- utf8mb4_unicode_ci DEALLOCATE test_merge_collation; -- -- -- -- PBE with explicit collation, PREPARE test_merge_collation(text) AS SELECT CONCAT($1 collate gbk_chinese_ci, futf8_bin) result, collation for(result) FROM t_diff_charset_columns; -- $1 use collation_connection, ERROR -DEALLOCATE test_merge_collation; +-- -- -- -- test revalidate +SELECT fgbk_chi result FROM t_diff_charset_columns WHERE fgbk_chi=_utf8mb4'高斯db'; -- 1 rows +PREPARE test_revalidate(text) AS +SELECT fgbk_chi result FROM t_diff_charset_columns WHERE fgbk_chi=$1; +EXECUTE test_revalidate(_utf8mb4'高斯db'); -- fgbk_chi collation has priority, 1 rows +ALTER INDEX idx_prefixkey_futf8_bin UNUSABLE; +EXECUTE test_revalidate(_utf8mb4'高斯db'); -- fgbk_chi collation has priority, 1 rows +SET NAMES utf8mb4 COLLATE utf8mb4_bin; +EXECUTE test_revalidate(_utf8mb4'高斯db'); -- fgbk_chi collation has priority, 1 rows +ALTER INDEX idx_prefixkey_futf8_bin REBUILD; +EXECUTE test_revalidate(_utf8mb4'高斯db'); -- fgbk_chi collation has priority, 1 rows +SET NAMES gbk COLLATE gbk_bin; +EXECUTE test_revalidate(_utf8mb4'高斯db'); -- fgbk_chi collation has priority, 1 rows +ALTER INDEX idx_prefixkey_futf8_bin REBUILD; +EXECUTE test_revalidate(_utf8mb4'高斯db'); -- fgbk_chi collation has priority, 1 rows +DEALLOCATE test_revalidate; +SET NAMES utf8mb4; -- -- concat column and PROCEDURE parameter with CURSOR -- -- -- implicit collation && string From b733e1bebee8c39f1d341a1f2edf78a4db3284bc Mon Sep 17 00:00:00 2001 From: gentle_hu Date: Sat, 26 Aug 2023 11:09:43 +0800 Subject: [PATCH 164/304] add testcase for statement_history and standby_statement_history --- src/test/regress/input/pg_proc_test.source | 8 ++++++++ src/test/regress/output/pg_proc_test.source | 11 +++++++++++ 2 files changed, 19 insertions(+) diff --git a/src/test/regress/input/pg_proc_test.source b/src/test/regress/input/pg_proc_test.source index 75e56d4c1..f6c20bbac 100644 --- a/src/test/regress/input/pg_proc_test.source +++ b/src/test/regress/input/pg_proc_test.source @@ -1,2 +1,10 @@ select provariadic,oid, proname from pg_proc where arraycontains(proargmodes::char[],ARRAY['v'::char]) order by oid; select prokind,length(prokind),count(*) from pg_proc where oid < 16384 group by prokind; + +-- The structure of function standby_statement_history must be consistent with table statement_history. +select + (select relnatts from pg_class where relname = 'statement_history' limit 1) + = + (select array_length(proargnames, 1) - 1 from pg_proc where proname = 'standby_statement_history' order by 1 limit 1) + as issame; + diff --git a/src/test/regress/output/pg_proc_test.source b/src/test/regress/output/pg_proc_test.source index 195b72bca..09411481c 100644 --- a/src/test/regress/output/pg_proc_test.source +++ b/src/test/regress/output/pg_proc_test.source @@ -34,3 +34,14 @@ select prokind,length(prokind),count(*) from pg_proc where oid < 16384 group by --? f | 1 | .* (1 row) +-- The structure of function standby_statement_history must be consistent with table statement_history. +select + (select relnatts from pg_class where relname = 'statement_history' limit 1) + = + (select array_length(proargnames, 1) - 1 from pg_proc where proname = 'standby_statement_history' order by 1 limit 1) + as issame; + issame +-------- + t +(1 row) + From 3031bff73aa676a5d11eb3bc7f3739a3199743b4 Mon Sep 17 00:00:00 2001 From: chenxiaobin19 <1025221611@qq.com> Date: Sat, 26 Aug 2023 15:52:31 +0800 Subject: [PATCH 165/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=8F=91=E5=B8=83?= =?UTF-8?q?=E8=AE=A2=E9=98=85=E5=9F=BA=E7=A1=80=E6=95=B0=E6=8D=AE=E5=A4=8D?= =?UTF-8?q?=E5=88=B6=E6=97=B6=E5=BE=80=E9=9D=9E=E7=A9=BA=E5=88=97=E4=BC=A0?= =?UTF-8?q?=E7=A9=BA=E5=80=BC=E7=9A=84core=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../storage/replication/logical/tablesync.cpp | 10 +++- src/test/subscription/schedule | 3 +- src/test/subscription/testcase/bugs.sh | 59 +++++++++++++++++++ 3 files changed, 70 insertions(+), 2 deletions(-) diff --git a/src/gausskernel/storage/replication/logical/tablesync.cpp b/src/gausskernel/storage/replication/logical/tablesync.cpp index 11484ae5d..5ea58c2e3 100644 --- a/src/gausskernel/storage/replication/logical/tablesync.cpp +++ b/src/gausskernel/storage/replication/logical/tablesync.cpp @@ -776,7 +776,15 @@ static void copy_table(Relation rel) /* Create CopyState for ingestion of the data from publisher. */ attnamelist = make_copy_attnamelist(relmapentry); - cstate = BeginCopyFrom(rel, NULL, attnamelist, NIL, &mem_info, (const char*)cmd.data, copy_read_data); + cstate = BeginCopyFrom(rel, NULL, attnamelist, NIL, &mem_info, NULL, copy_read_data); + + RangeTblEntry *rte = makeNode(RangeTblEntry); + rte->rtekind = RTE_RELATION; + rte->relid = RelationGetRelid(rel); + rte->relkind = rel->rd_rel->relkind; + rte->requiredPerms = ACL_SELECT; + + cstate->range_table = list_make1(rte); /* Do the copy */ (void)CopyFrom(cstate); diff --git a/src/test/subscription/schedule b/src/test/subscription/schedule index 836ac0e58..0635e0941 100644 --- a/src/test/subscription/schedule +++ b/src/test/subscription/schedule @@ -13,4 +13,5 @@ matviews change_wal_level skiplsn disable -pub_subconflict \ No newline at end of file +pub_subconflict +bugs \ No newline at end of file diff --git a/src/test/subscription/testcase/bugs.sh b/src/test/subscription/testcase/bugs.sh index e69de29bb..ec35e5473 100644 --- a/src/test/subscription/testcase/bugs.sh +++ b/src/test/subscription/testcase/bugs.sh @@ -0,0 +1,59 @@ +#!/bin/sh + +source $1/env_utils.sh $1 $2 + +case_db="bugs_db" + +function test_1() { + echo "create database and tables." + exec_sql $db $pub_node1_port "CREATE DATABASE $case_db" + exec_sql $db $sub_node1_port "CREATE DATABASE $case_db" + # Create some preexisting content on publisher + exec_sql $case_db $pub_node1_port "CREATE TABLE tab_rep (a int primary key, b text)" + exec_sql $case_db $pub_node1_port "INSERT INTO tab_rep VALUES (1)" + + # Setup structure on subscriber + exec_sql $case_db $sub_node1_port "CREATE TABLE tab_rep (a int primary key, b text not null default 0)" + + # Setup logical replication + echo "create publication and subscription." + publisher_connstr="port=$pub_node1_port host=$g_local_ip dbname=$case_db user=$username password=$passwd" + exec_sql $case_db $pub_node1_port "CREATE PUBLICATION tap_pub FOR ALL TABLES" + exec_sql $case_db $sub_node1_port "CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr' PUBLICATION tap_pub" + + logfile=$(get_log_file "sub_datanode1") + + location=$(awk 'END{print NR}' $logfile) + + content=$(tail -n +$location $logfile) + targetstr=$(expr "$content" : '.*\(Failing row contains\).*') + + attempt=0 + while [ -z "$targetstr" ] + do + content=$(tail -n +$location $logfile) + targetstr=$(expr "$content" : '.*\(Failing row contains\).*') + attempt=`expr $attempt \+ 1` + + sleep 1 + if [ $attempt -eq 5 ]; then + echo "$failed_keyword when check failing row log" + exit 1 + fi + done + + echo "check failing row log success" +} + +function tear_down() { + exec_sql $case_db $sub_node1_port "DROP SUBSCRIPTION IF EXISTS tap_sub" + exec_sql $case_db $pub_node1_port "DROP PUBLICATION IF EXISTS tap_pub" + + exec_sql $db $sub_node1_port "DROP DATABASE $case_db" + exec_sql $db $pub_node1_port "DROP DATABASE $case_db" + + echo "tear down" +} + +test_1 +tear_down \ No newline at end of file From 920b7eefaf0684d1697c5bf57332b2d80e2574bd Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Sat, 26 Aug 2023 16:01:06 +0800 Subject: [PATCH 166/304] =?UTF-8?q?8.26dss=E6=8E=A8=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/ddes/ddes_commit_id | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gausskernel/ddes/ddes_commit_id b/src/gausskernel/ddes/ddes_commit_id index 514b56258..31dab8b27 100644 --- a/src/gausskernel/ddes/ddes_commit_id +++ b/src/gausskernel/ddes/ddes_commit_id @@ -1,3 +1,3 @@ dms_commit_id=19301cb7b3dd7f959cf6269c6d2aa8c8f23804fa -dss_commit_id=45ae7916cbdda2b2e64c02811db30df565cf34fa +dss_commit_id=d48ae4eaa803bb9dec414b25f26ba37fff8942df cbb_commit_id=41cadda4643656bcd856bd37d2b6908e44a5d46c \ No newline at end of file From 194643b6ccfb0dcf53d7889621b966c26f0b01c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BD=98=E5=85=B4=E5=BD=AC?= Date: Sat, 26 Aug 2023 16:45:16 +0800 Subject: [PATCH 167/304] =?UTF-8?q?memcpy=5Fs=E5=90=8E=E5=AE=89=E5=85=A8?= =?UTF-8?q?=E6=A3=80=E6=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/utils/cache/knl_globalpartdefcache.cpp | 3 ++- src/gausskernel/process/postmaster/pagerepair.cpp | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/common/backend/utils/cache/knl_globalpartdefcache.cpp b/src/common/backend/utils/cache/knl_globalpartdefcache.cpp index 4573ac53a..00412ad71 100644 --- a/src/common/backend/utils/cache/knl_globalpartdefcache.cpp +++ b/src/common/backend/utils/cache/knl_globalpartdefcache.cpp @@ -50,7 +50,8 @@ void CopyPartitionData(Partition dest_partition, Partition src_partition) /* We just copy fixed field */ dest_partition->pd_part = (Form_pg_partition)palloc(PARTITION_TUPLE_SIZE); - memcpy_s(dest_partition->pd_part, PARTITION_TUPLE_SIZE, src_partition->pd_part, PARTITION_TUPLE_SIZE); + errno_t rc = memcpy_s(dest_partition->pd_part, PARTITION_TUPLE_SIZE, src_partition->pd_part, PARTITION_TUPLE_SIZE); + securec_check(rc, "\0", "\0"); dest_partition->pd_smgr = NULL; Assert(src_partition->pd_isvalid); diff --git a/src/gausskernel/process/postmaster/pagerepair.cpp b/src/gausskernel/process/postmaster/pagerepair.cpp index ac2815fe9..4e4556098 100644 --- a/src/gausskernel/process/postmaster/pagerepair.cpp +++ b/src/gausskernel/process/postmaster/pagerepair.cpp @@ -269,7 +269,7 @@ void CopyPageToRepairHashTbl(RepairBlockEntry *entry, char *page_content) XLogRecPtr page_lsn = PageGetLSN(page_content); errno_t rc = 0; - memcpy_s(entry->page_content, BLCKSZ, page_content, BLCKSZ); + rc = memcpy_s(entry->page_content, BLCKSZ, page_content, BLCKSZ); securec_check(rc, "", ""); if (entry->error_type == CRC_CHECK_FAIL) { From bbfedd4e677ea1deb22c08f38579174398c2fd98 Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Sat, 26 Aug 2023 18:13:27 +0800 Subject: [PATCH 168/304] =?UTF-8?q?ctlinfo=E6=A0=A1=E9=AA=8C=E5=A4=B1?= =?UTF-8?q?=E8=B4=A5=E8=A7=A3=E5=86=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../replication/ss_cluster_replication.cpp | 35 +++++++++++-------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/src/gausskernel/storage/replication/ss_cluster_replication.cpp b/src/gausskernel/storage/replication/ss_cluster_replication.cpp index b3fb10268..a60d52255 100644 --- a/src/gausskernel/storage/replication/ss_cluster_replication.cpp +++ b/src/gausskernel/storage/replication/ss_cluster_replication.cpp @@ -67,12 +67,13 @@ void WriteSSDoradoCtlInfoFile() void ReadSSDoradoCtlInfoFile() { struct stat st; - Assert(stat(SS_DORADO_CTRL_FILE, &st) == 0 && S_ISREG(st.st_mode)); - Assert(g_instance.xlog_cxt.ssReplicationXLogCtl != NULL); - ShareStorageXLogCtl *ctlInfo = g_instance.xlog_cxt.ssReplicationXLogCtl; - errno_t errorno = EOK; - int fd = -1; - fd = BasicOpenFile(SS_DORADO_CTRL_FILE, O_RDWR | PG_BINARY, S_IRUSR | S_IWUSR); + if (!stat(SS_DORADO_CTRL_FILE, &st) == 0 || !S_ISREG(st.st_mode)) { + ereport(PANIC, + (errcode_for_file_access(), errmsg("[ReadSSDoradoCtlInfo]SS dorado control file is not exist\"%s\".", + SS_DORADO_CTRL_FILE))); + } + + int fd = BasicOpenFile(SS_DORADO_CTRL_FILE, O_RDWR | PG_BINARY, S_IRUSR | S_IWUSR); if (fd < 0) { /* TODO: need consider that dorado is briefly unreadable during the synchronization process */ ereport(PANIC, @@ -82,27 +83,31 @@ void ReadSSDoradoCtlInfoFile() char buffer[SS_DORADO_CTL_INFO_SIZE] __attribute__((__aligned__(ALIGNOF_BUFFER))); if (read(fd, buffer, SS_DORADO_CTL_INFO_SIZE) != SS_DORADO_CTL_INFO_SIZE) { + (void)close(fd); ereport(PANIC, (errcode_for_file_access(), errmsg("[ReadSSDoradoCtlInfo]could not read SS dorado control file \"%s\".", SS_DORADO_CTRL_FILE))); } - errorno = memcpy_s(ctlInfo, SS_DORADO_CTL_INFO_SIZE, buffer, SS_DORADO_CTL_INFO_SIZE); - securec_check_c(errorno, "\0", "\0"); - if (close(fd)) { - ereport(PANIC, - (errcode_for_file_access(), errmsg("[ReadSSDoradoCtlInfo]could not close SS dorado control file \"%s\".", - SS_DORADO_CTRL_FILE))); - } - + ShareStorageXLogCtl *ctlInfo = (ShareStorageXLogCtl*)buffer; if ((ctlInfo->magic != SHARE_STORAGE_CTL_MAGIC) || (ctlInfo->checkNumber != SHARE_STORAGE_CTL_CHCK_NUMBER)) { + (void)close(fd); ereport(FATAL, (errmsg("[ReadSSDoradoCtlInfo]SS replication ctl_info maybe damaged."))); } pg_crc32c crc = CalShareStorageCtlInfoCrc(ctlInfo); if (!EQ_CRC32C(crc, ctlInfo->crc)) { + (void)close(fd); ereport(FATAL, (errmsg("[ReadSSDoradoCtlInfo]SS replication ctl_info crc check failed."))); } + + errno_t errorno = memcpy_s(g_instance.xlog_cxt.ssReplicationXLogCtl, SS_DORADO_CTL_INFO_SIZE, buffer, SS_DORADO_CTL_INFO_SIZE); + securec_check_c(errorno, "\0", "\0"); + if (close(fd)) { + ereport(PANIC, + (errcode_for_file_access(), errmsg("[ReadSSDoradoCtlInfo]could not close SS dorado control file \"%s\".", + SS_DORADO_CTRL_FILE))); + } } void InitSSDoradoCtlInfoFile() @@ -120,7 +125,7 @@ void InitSSDoradoCtlInfoFile() int fd = -1; char buffer[SS_DORADO_CTL_INFO_SIZE] __attribute__((__aligned__(ALIGNOF_BUFFER))); /* need to be aligned */ errno_t errorno = EOK; - Assert(stat(SS_DORADO_CTRL_FILE, &st) !=0 || !S_ISREG(st.st_mode)); + Assert(stat(SS_DORADO_CTRL_FILE, &st) != 0 || !S_ISREG(st.st_mode)); /* create SS_DORADO_CTRL_FILE first time */ fd = BasicOpenFile(SS_DORADO_CTRL_FILE, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, S_IRUSR | S_IWUSR); From 0a0808fb462ee7cac712ef4cac147e078b51cda0 Mon Sep 17 00:00:00 2001 From: "arcoalien@qq.com" Date: Mon, 21 Aug 2023 15:41:27 +0800 Subject: [PATCH 169/304] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E5=A4=87=E6=9C=BA?= =?UTF-8?q?=E5=BC=82=E5=B8=B8=E8=A7=A6=E5=8F=91reform=E5=90=8E=EF=BC=8C?= =?UTF-8?q?=E4=B8=BB=E6=9C=BA=E8=B7=91=E4=B8=9A=E5=8A=A1=E4=BC=9A=E5=8D=A1?= =?UTF-8?q?=E6=AD=BB=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ddes/adapter/ss_dms_bufmgr.cpp | 2 +- .../ddes/adapter/ss_dms_callback.cpp | 6 +- .../storage/smgr/segment/data_file.cpp | 35 +++- .../storage/smgr/segment/extent_group.cpp | 4 +- .../storage/smgr/segment/space.cpp | 156 ++++++++++++++++++ src/include/ddes/dms/ss_common_attr.h | 10 ++ src/include/storage/smgr/segment.h | 1 + 7 files changed, 201 insertions(+), 13 deletions(-) diff --git a/src/gausskernel/ddes/adapter/ss_dms_bufmgr.cpp b/src/gausskernel/ddes/adapter/ss_dms_bufmgr.cpp index 917f63303..2dcaae232 100644 --- a/src/gausskernel/ddes/adapter/ss_dms_bufmgr.cpp +++ b/src/gausskernel/ddes/adapter/ss_dms_bufmgr.cpp @@ -944,5 +944,5 @@ long SSGetBufSleepTime(int retry_times) if (retry_times < ss_buf_retry_threshold) { return 5000L * retry_times; } - return 1000L * 1000 * 60; + return 1000L * 1000 * 20; } diff --git a/src/gausskernel/ddes/adapter/ss_dms_callback.cpp b/src/gausskernel/ddes/adapter/ss_dms_callback.cpp index 24cf0ffd5..e7cb2ad11 100644 --- a/src/gausskernel/ddes/adapter/ss_dms_callback.cpp +++ b/src/gausskernel/ddes/adapter/ss_dms_callback.cpp @@ -1327,7 +1327,8 @@ static int CBConfirmOwner(void *db_handle, char *pageid, unsigned char *lock_mod SSGetBufferDesc(pageid, &valid, &buf_desc); if (buf_desc == NULL) { - return DMS_ERROR; + *lock_mode = (uint8)DMS_LOCK_NULL; + return GS_SUCCESS; } if (!valid) { @@ -1364,7 +1365,8 @@ static int CBConfirmConverting(void *db_handle, char *pageid, unsigned char smon SSGetBufferDesc(pageid, &valid, &buf_desc); if (buf_desc == NULL) { - return DMS_ERROR; + *lock_mode = (uint8)DMS_LOCK_NULL; + return GS_SUCCESS; } if (!valid) { diff --git a/src/gausskernel/storage/smgr/segment/data_file.cpp b/src/gausskernel/storage/smgr/segment/data_file.cpp index f20810f47..523c83f3b 100644 --- a/src/gausskernel/storage/smgr/segment/data_file.cpp +++ b/src/gausskernel/storage/smgr/segment/data_file.cpp @@ -61,19 +61,24 @@ static int dv_open_file(char *filename, uint32 flags, int mode) static void dv_close_file(int fd) { - close(fd); + if (fd > 0) { + close(fd); + } ereport(LOG, (errmsg("dv_close_file fd is %d", fd))); } /* Return a palloc string, and callers should free it */ static char *slice_filename(const char *filename, int sliceno) { - char *res = (char *)palloc(MAXPGPATH); + char *res = NULL; + int len = strlen(filename); if (sliceno == 0) { - errno_t rc = snprintf_s(res, MAXPGPATH, MAXPGPATH - 1, "%s", filename); + res = (char *)palloc(len + 1); + errno_t rc = sprintf_s(res, len + 1, "%s", filename); securec_check_ss(rc, "\0", "\0"); } else { - errno_t rc = snprintf_s(res, MAXPGPATH, MAXPGPATH - 1, "%s.%d", filename, sliceno); + res = (char *)palloc(len + MAX_LEN_OF_MAXINTRANGE); + errno_t rc = sprintf_s(res, len + MAX_LEN_OF_MAXINTRANGE, "%s.%d", filename, sliceno); securec_check_ss(rc, "\0", "\0"); } return res; @@ -154,28 +159,34 @@ bool df_ss_update_segfile_size(SegLogicFile *sf, BlockNumber target_block) char *filename = slice_filename(sf->filename, 0); int fd = dv_open_file(filename, flags, (int)SEGMENT_FILE_MODE); if (fd < 0) { + pfree(filename); ereport(LOG, (errmodule(MOD_SEGMENT_PAGE), errmsg("File \"%s\" does not exist, stop read here.", filename))); - pfree(filename); return false; } sf->file_num++; sf->segfiles[0].fd = fd; sf->segfiles[0].sliceno = 0; + pfree(filename); } - int sliceno = sf->file_num - 1; int fd = sf->segfiles[sliceno].fd; + if (fd <= 0) { + char *filename = slice_filename(sf->filename, sliceno); + sf->segfiles[sliceno].fd = dv_open_file(filename, flags, SEGMENT_FILE_MODE); + fd = sf->segfiles[sliceno].fd; + pfree(filename); + } off_t size = lseek(fd, 0L, SEEK_END); sf->total_blocks = (uint32)(sliceno * DF_FILE_SLICE_BLOCKS + size / BLCKSZ); /* size of full slices + last slice */ while (size == DF_FILE_SLICE_SIZE) { sliceno = sf->file_num; - char *filename = slice_filename(sf->filename, sf->file_num); /* needed if primary created new slice */ if (sliceno >= sf->vector_capacity) { df_extend_file_vector(sf); } + char *filename = slice_filename(sf->filename, sliceno); fd = dv_open_file(filename, flags, (int)SEGMENT_FILE_MODE); if (fd < 0) { ereport(LOG, @@ -190,12 +201,13 @@ bool df_ss_update_segfile_size(SegLogicFile *sf, BlockNumber target_block) size = lseek(fd, 0L, SEEK_END); sf->total_blocks += (uint32)(size / BLCKSZ); sf->file_num++; + pfree(filename); } if (sf->total_blocks <= target_block) { return false; } - + return true; } @@ -220,6 +232,11 @@ SegPhysicalFile df_get_physical_file(SegLogicFile *sf, int sliceno, BlockNumber } SegmentCheck(sliceno < sf->file_num); + if (SS_STANDBY_MODE && sf->segfiles[sliceno].fd <= 0) { + char *filename = slice_filename(sf->filename, sliceno); + sf->segfiles[sliceno].fd = dv_open_file(filename, O_RDONLY | PG_BINARY, SEGMENT_FILE_MODE); + pfree(filename); + } SegPhysicalFile spf = sf->segfiles[sliceno]; return spf; @@ -258,7 +275,7 @@ void df_extend_file_vector(SegLogicFile *sf) { int new_capacity = sf->vector_capacity + DF_ARRAY_EXTEND_STEP; MemoryContext oldcnxt = MemoryContextSwitchTo(INSTANCE_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_STORAGE)); - SegPhysicalFile *newfiles = (SegPhysicalFile *)palloc(sizeof(SegPhysicalFile) * new_capacity); + SegPhysicalFile *newfiles = (SegPhysicalFile *)palloc0(sizeof(SegPhysicalFile) * new_capacity); MemoryContextSwitchTo(oldcnxt); for (int i = 0; i < sf->file_num; i++) { diff --git a/src/gausskernel/storage/smgr/segment/extent_group.cpp b/src/gausskernel/storage/smgr/segment/extent_group.cpp index b472a35db..ddfb334a2 100644 --- a/src/gausskernel/storage/smgr/segment/extent_group.cpp +++ b/src/gausskernel/storage/smgr/segment/extent_group.cpp @@ -109,7 +109,9 @@ void eg_init_df_ctrl(SegExtentGroup *seg) SegLogicFile *sf = (SegLogicFile *)palloc(sizeof(SegLogicFile)); MemoryContextSwitchTo(oldcnxt); df_ctrl_init(sf, seg->rnode, seg->forknum); - df_open_files(sf); + if (!SS_STANDBY_MODE) { + df_open_files(sf); + } seg->segfile = sf; } diff --git a/src/gausskernel/storage/smgr/segment/space.cpp b/src/gausskernel/storage/smgr/segment/space.cpp index 9de167b89..122ff7884 100644 --- a/src/gausskernel/storage/smgr/segment/space.cpp +++ b/src/gausskernel/storage/smgr/segment/space.cpp @@ -43,6 +43,8 @@ #include "storage/file/fio_device.h" #include "libaio.h" +static void SSInitSegLogicFile(SegSpace *spc); + void spc_lock(SegSpace *spc) { PthreadMutexLock(t_thrd.utils_cxt.CurrentResourceOwner, &spc->lock, true); @@ -241,6 +243,10 @@ void InitSpaceNode(SegSpace *spc, Oid spcNode, Oid dbNode, bool is_redo) eg_ctrl_init(spc, &spc->extent_group[egid][forknum], EXTENT_GROUPID_TO_SIZE(egid), forknum); } } + + if (SS_STANDBY_MODE) { + SSInitSegLogicFile(spc); + } } void spc_clean_extent_groups(SegSpace *spc) @@ -1353,3 +1359,153 @@ void InitSegSpcCache(void) } } +static bool SSCheckIfSegLogicFileNormal(SegExtentGroup *seg) +{ + SegLogicFile *sf = seg->segfile; + if (sf->total_blocks < DF_FILE_MIN_BLOCKS) { + return false; + } + + int fd = BasicOpenFile(sf->filename, O_RDWR | PG_BINARY, S_IWUSR | S_IRUSR); + if (fd < 0) { + ereport(ERROR, (errmsg("open_file failed filename: %s, fd is %d, %d", sf->filename, fd, errno))); + } + sf->segfiles[0].fd = fd; + char* buffer = (char *)palloc(BLCKSZ + ALIGNOF_BUFFER); + char* aligned_buffer = (char *)BUFFERALIGN(buffer); + int nbytes = pread(fd, aligned_buffer, BLCKSZ, DF_MAP_HEAD_PAGE * BLCKSZ); + if (nbytes != BLCKSZ) { + ereport(ERROR, (errmsg("could not read segment meta block in file %s, %d", sf->filename, errno))); + } + + if (!PageIsVerified((Page)aligned_buffer, DF_MAP_HEAD_PAGE)) { + pfree(buffer); + return false; + } + + df_map_head_t *map_head = (df_map_head_t *)PageGetContents((Page)aligned_buffer); + if (map_head->bit_unit != seg->extent_size) { + pfree(buffer); + return false; + } + + pfree(buffer); + return true; +} + +static void SSUpdateSegLogicFileSize(SegSpace *spc) +{ + bool is_normal = true; + bool is_meta_normal = false; + + for (int egid = 0; egid < EXTENT_GROUPS; egid++) { + for (int forknum = 0; forknum <= SEGMENT_MAX_FORKNUM; forknum++) { + SegLogicFile *sf = spc->extent_group[egid][forknum].segfile; + if (sf->file_num == 0) { + continue; + } + + struct stat statbuf; + if (sf->file_num == 1) { + if (stat(sf->filename, &statbuf) == 0) { + sf->total_blocks = statbuf.st_size / BLCKSZ; + } else { + ereport(ERROR, (errmsg("failed stat file %s during init segment file.", sf->filename))); + } + } else { + char fullpath[MAXPGPATH]; + errno_t rc = sprintf_s(fullpath, MAXPGPATH, "%s.%d", sf->filename, sf->file_num - 1); + securec_check_ss(rc, "\0", "\0"); + if (stat(fullpath, &statbuf) == 0) { + sf->total_blocks = statbuf.st_size / BLCKSZ + (sf->file_num - 1) * EXT_SIZE_1024_TOTAL_PAGES; + } else { + ereport(ERROR, (errmsg("failed stat file %s during init segment file.", fullpath))); + } + } + + if (!is_normal) { + continue; + } + + /* we can set spc status to open here, only need to open sf->filename and read one block to verify */ + if (is_normal && SSCheckIfSegLogicFileNormal(&(spc->extent_group[egid][forknum]))) { + if (egid == 0 && forknum == 0) { + is_meta_normal = true; + } + } else { + is_normal = false; + } + } + } + + if (is_meta_normal && is_normal) { + spc->status = OPENED; + } +} + +static void SSUpdateSegLogicFileNum(SegLogicFile* sf, char* dirpath, char* filename) +{ + int sliceno = sf->file_num + 1; + if (sliceno > sf->vector_capacity) { + df_extend_file_vector(sf); + } + sf->segfiles[sf->file_num].sliceno = sf->file_num; + sf->file_num++; +} + +static void SSInitSegLogicFile(SegSpace *spc) +{ + if (spc->extent_group[0][0].segfile == NULL) { + return; + } + SegmentCheck(spc->extent_group[0][0].segfile->filename[0] != '\0'); + /* Get path of dir from seg filename */ + char dirpath[MAXPGPATH]; + int count = strlen(spc->extent_group[0][0].segfile->filename) - SEG_MAINFORK_FILENAME_LEN; + int rc = EOK; + rc = strncpy_s(dirpath, MAXPGPATH, spc->extent_group[0][0].segfile->filename, count); + securec_check_c(rc, "\0", "\0"); + + /* + * Read dir and fill seg logic file except fd. + * For filenum and total block, we only need to check the filename and size under the dir. + * For fd, we can construct the filename and open it when we really need use the file. + */ + DIR *data_dir = NULL; + struct dirent *data_de = NULL; + data_dir = opendir(dirpath); + if (data_dir == NULL) { + ereport(ERROR, + (errcode_for_file_access(), errmsg("could not open data dir %s during init segment file.", dirpath))); + } + + while ((data_de = readdir(data_dir)) != NULL) { + if (!isdigit(data_de->d_name[0])) { + continue; + } + + char tmp_path[MAXPGPATH]; + int suffix = 0; + rc = sscanf_s(data_de->d_name, "%[^.].%d", tmp_path, MAXPGPATH, &suffix); + if (rc <= 0) { + ereport(LOG, (errmsg("skip %s as it is not segment file.", data_de->d_name))); + continue; + } + int extent_size = tmp_path[0] - '0'; + int tmp_length = strlen(tmp_path); + if (strstr(tmp_path, "_vm") != NULL && tmp_length == SEG_VMFORK_FILENAME_LEN && extent_size >= EXTENT_1 && + extent_size <= EXTENT_8192) { + SSUpdateSegLogicFileNum(spc->extent_group[extent_size - 1][VISIBILITYMAP_FORKNUM].segfile, dirpath, + data_de->d_name); + } else if (strstr(tmp_path, "_fsm") != NULL && tmp_length == SEG_FSMFORK_FILENAME_LEN && + extent_size >= EXTENT_1 && extent_size <= EXTENT_8192) { + SSUpdateSegLogicFileNum(spc->extent_group[extent_size - 1][FSM_FORKNUM].segfile, dirpath, data_de->d_name); + } else if (tmp_length == 1 && extent_size >= EXTENT_1 && extent_size <= EXTENT_8192) { + SSUpdateSegLogicFileNum(spc->extent_group[extent_size - 1][MAIN_FORKNUM].segfile, dirpath, data_de->d_name); + } else { + ereport(LOG, (errmsg("skip %s as it is not segment file.", data_de->d_name))); + } + } + SSUpdateSegLogicFileSize(spc); + closedir(data_dir); +} diff --git a/src/include/ddes/dms/ss_common_attr.h b/src/include/ddes/dms/ss_common_attr.h index 59b933d2f..413a33528 100644 --- a/src/include/ddes/dms/ss_common_attr.h +++ b/src/include/ddes/dms/ss_common_attr.h @@ -191,6 +191,16 @@ #define DMS_MSG_MAX_WAIT_TIME (10 * 1000) // 10s #define SS_REFORM_WAIT_TIME (5000) // 5ms +/* length of segment filename like '/1' */ +#define SEG_MAINFORK_FILENAME_LEN 2 +/* length of segment vm filename like '1_vm' */ +#define SEG_VMFORK_FILENAME_LEN 4 +/* length of segment fsm filename like '1_fsm' */ +#define SEG_FSMFORK_FILENAME_LEN 5 + +/* max length of max int range as char */ +#define MAX_LEN_OF_MAXINTRANGE 12 + typedef enum SSBroadcastOp { BCAST_CANCEL_TRX_FOR_SWITCHOVER = 0, BCAST_SI, diff --git a/src/include/storage/smgr/segment.h b/src/include/storage/smgr/segment.h index 578cfea32..8cb73ac3e 100644 --- a/src/include/storage/smgr/segment.h +++ b/src/include/storage/smgr/segment.h @@ -67,6 +67,7 @@ int seg_sync_filetag(const FileTag *ftag, char *path); int seg_unlink_filetag(const FileTag *ftag, char *path); void segForgetDatabaseFsyncRequests(Oid dbid); bool seg_filetag_matches(const FileTag *ftag, const FileTag *candidate); +void df_extend_file_vector(SegLogicFile *sf); /* * XLog Atomic Operation APIs From d6b02e3f9a1714f5f7d646ef399e740255ad771b Mon Sep 17 00:00:00 2001 From: weiwentao <1375910710@qq.com> Date: Sat, 26 Aug 2023 10:15:39 +0800 Subject: [PATCH 170/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dcollation=5Fconnectio?= =?UTF-8?q?n=E5=8F=AF=E4=BB=A5=E8=AE=BE=E7=BD=AE=E7=B3=BB=E7=BB=9F?= =?UTF-8?q?=E5=AD=97=E7=AC=A6=E9=9B=86=E7=9A=84=E9=97=AE=E9=A2=98=EF=BC=8C?= =?UTF-8?q?=E7=9B=AE=E5=89=8D=E5=8F=AA=E8=83=BD=E8=AE=BE=E7=BD=AEM*?= =?UTF-8?q?=E7=9A=84=E5=AD=97=E7=AC=A6=E9=9B=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/optimizer/commands/variable.cpp | 2 +- .../regress/output/charset_connection_test.source | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/gausskernel/optimizer/commands/variable.cpp b/src/gausskernel/optimizer/commands/variable.cpp index 5f5acd1d7..3cdc8c697 100644 --- a/src/gausskernel/optimizer/commands/variable.cpp +++ b/src/gausskernel/optimizer/commands/variable.cpp @@ -874,7 +874,7 @@ bool check_collation_connection(char** newval, void** extra, GucSource source) } Oid collid = get_collation_oid_with_lower_name(*newval, PG_INVALID_ENCODING); - if (!OidIsValid(collid)) { + if (!COLLATION_IN_B_FORMAT(collid)) { return false; } diff --git a/src/test/regress/output/charset_connection_test.source b/src/test/regress/output/charset_connection_test.source index 6478f6da4..583ea3664 100644 --- a/src/test/regress/output/charset_connection_test.source +++ b/src/test/regress/output/charset_connection_test.source @@ -293,16 +293,17 @@ show collation_connection; set names gbk; set names utf8 collate 'default'; +ERROR: invalid value for parameter "collation_connection": "default" show character_set_connection; character_set_connection -------------------------- - UTF8 + GBK (1 row) show collation_connection; collation_connection ---------------------- - default + gbk_chinese_ci (1 row) set names gbk; @@ -321,6 +322,7 @@ show collation_connection; -- test charset = -1 set collation_connection = 'C'; +ERROR: invalid value for parameter "collation_connection": "C" show character_set_connection; character_set_connection -------------------------- @@ -330,10 +332,11 @@ show character_set_connection; show collation_connection; collation_connection ---------------------- - C + utf8mb4_general_ci (1 row) set collation_connection = 'default'; +ERROR: invalid value for parameter "collation_connection": "default" show character_set_connection; character_set_connection -------------------------- @@ -343,10 +346,11 @@ show character_set_connection; show collation_connection; collation_connection ---------------------- - default + utf8mb4_general_ci (1 row) set collation_connection = 'POSIX'; +ERROR: invalid value for parameter "collation_connection": "POSIX" show character_set_connection; character_set_connection -------------------------- @@ -356,7 +360,7 @@ show character_set_connection; show collation_connection; collation_connection ---------------------- - POSIX + utf8mb4_general_ci (1 row) \! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1 -c "b_format_behavior_compat_options=''" >/dev/null 2>&1 From a6720f2a11cf0629b8968b5dae4476ac4b685655 Mon Sep 17 00:00:00 2001 From: luozihao <1165977584@qq.com> Date: Mon, 28 Aug 2023 20:00:36 +0800 Subject: [PATCH 171/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dinterval=E7=B1=BB?= =?UTF-8?q?=E5=9E=8B=E5=9C=A8=E5=AD=98=E5=82=A8=E8=BF=87=E7=A8=8B=E5=A4=8D?= =?UTF-8?q?=E5=88=B6=E6=B5=81=E7=A8=8B=E4=B8=AD=E8=B5=8B=E5=80=BC=E6=8A=A5?= =?UTF-8?q?=E9=94=99=E7=9A=84bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/pl/plpgsql/src/pl_exec.cpp | 3 +- src/test/regress/expected/interval.out | 55 ++++++++++++++++++++++++++ src/test/regress/sql/interval.sql | 41 ++++++++++++++++++- 3 files changed, 97 insertions(+), 2 deletions(-) diff --git a/src/common/pl/plpgsql/src/pl_exec.cpp b/src/common/pl/plpgsql/src/pl_exec.cpp index 5e8bb96f3..f1de71665 100644 --- a/src/common/pl/plpgsql/src/pl_exec.cpp +++ b/src/common/pl/plpgsql/src/pl_exec.cpp @@ -12580,7 +12580,8 @@ static Datum exec_cast_value(PLpgSQL_execstate* estate, Datum value, Oid valtype /* get the implicit cast function from valtype to reqtype */ result = find_coercion_pathway(reqtype, valtype, COERCION_ASSIGNMENT, &funcid); if (funcid != InvalidOid && !(result == COERCION_PATH_COERCEVIAIO || result == COERCION_PATH_ARRAYCOERCE)) { - value = OidFunctionCall1(funcid, value); + value = (reqtype == INTERVALOID) ? + OidFunctionCall2(funcid, value, reqtypmod) : OidFunctionCall1(funcid, value); value = pl_coerce_type_typmod(value, reqtype, reqtypmod); } else { extval = convert_value_to_string(estate, value, valtype); diff --git a/src/test/regress/expected/interval.out b/src/test/regress/expected/interval.out index fbe5b481a..adc32e086 100644 --- a/src/test/regress/expected/interval.out +++ b/src/test/regress/expected/interval.out @@ -1310,3 +1310,58 @@ select '15'::bpchar(2)::INTERVAL YEAR; @ 15 years (1 row) +-- test abourt interval typmod in procedure +drop table if exists all_datatype_tbl; +NOTICE: table "all_datatype_tbl" does not exist, skipping +create table all_datatype_tbl( + c_id integer, + c_boolean boolean, + c_integer integer, c_bigint bigint, + c_real real, + c_decimal decimal(38), c_number number(38), + c_char char(50) default null, c_varchar varchar(50), c_clob clob, + c_blob blob, + c_timestamp timestamp, c_interval interval day to second) with (segment=on) ; +create or replace procedure pro_012() +as + sqlstat varchar(500); + v1 interval day to second; +begin + v1 := '12 12:3:4.1234'; + sqlstat := 'insert into all_datatype_tbl(c_interval) select :p1'; + execute immediate sqlstat using v1; +end; +/ +call pro_012(); + pro_012 +--------- + +(1 row) + +select c_interval from all_datatype_tbl; + c_interval +--------------------------------------- + @ 12 days 12 hours 3 mins 4.1234 secs +(1 row) + +drop procedure pro_012; +create or replace procedure pro_015() +as + sqlstat varchar(500); + v1 interval day to second; + r1 interval day to second; +begin + v1 := '12 12:3:4.1234'; + sqlstat := 'select :p1'; + execute immediate sqlstat into r1 using v1; + raise info 'result:%',v1; +end; +/ +call pro_015(); +INFO: result:@ 12 days 12 hours 3 mins 4.1234 secs + pro_015 +--------- + +(1 row) + +drop procedure pro_015; diff --git a/src/test/regress/sql/interval.sql b/src/test/regress/sql/interval.sql index 4b5f97bd1..58b9de659 100644 --- a/src/test/regress/sql/interval.sql +++ b/src/test/regress/sql/interval.sql @@ -367,4 +367,43 @@ select '15'::float8::INTERVAL YEAR; select '15'::numeric::INTERVAL YEAR; select '15'::text::INTERVAL YEAR; select '15'::varchar::INTERVAL YEAR; -select '15'::bpchar(2)::INTERVAL YEAR; \ No newline at end of file +select '15'::bpchar(2)::INTERVAL YEAR; + +-- test abourt interval typmod in procedure +drop table if exists all_datatype_tbl; +create table all_datatype_tbl( + c_id integer, + c_boolean boolean, + c_integer integer, c_bigint bigint, + c_real real, + c_decimal decimal(38), c_number number(38), + c_char char(50) default null, c_varchar varchar(50), c_clob clob, + c_blob blob, + c_timestamp timestamp, c_interval interval day to second) with (segment=on) ; +create or replace procedure pro_012() +as + sqlstat varchar(500); + v1 interval day to second; +begin + v1 := '12 12:3:4.1234'; + sqlstat := 'insert into all_datatype_tbl(c_interval) select :p1'; + execute immediate sqlstat using v1; +end; +/ +call pro_012(); +select c_interval from all_datatype_tbl; +drop procedure pro_012; +create or replace procedure pro_015() +as + sqlstat varchar(500); + v1 interval day to second; + r1 interval day to second; +begin + v1 := '12 12:3:4.1234'; + sqlstat := 'select :p1'; + execute immediate sqlstat into r1 using v1; + raise info 'result:%',v1; +end; +/ +call pro_015(); +drop procedure pro_015; From 459ec58a585ed8ba9a547fafd1b0d5b6bff8b2b5 Mon Sep 17 00:00:00 2001 From: chendong76 <1209756284@qq.com> Date: Tue, 29 Aug 2023 16:43:16 +0800 Subject: [PATCH 172/304] =?UTF-8?q?=E3=80=90bugfix=E3=80=91=E8=A7=A3?= =?UTF-8?q?=E5=86=B3=E9=83=A8=E5=88=86Switchover=E5=9C=BA=E6=99=AF?= =?UTF-8?q?=E4=B8=8B=EF=BC=8C=E9=A2=84=E6=9C=9F=E5=A4=96=E8=BF=9B=E5=85=A5?= =?UTF-8?q?=E6=8C=89=E9=9C=80=E5=9B=9E=E6=94=BE=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/storage/access/transam/xlog.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/gausskernel/storage/access/transam/xlog.cpp b/src/gausskernel/storage/access/transam/xlog.cpp index 3a85240b5..b5720651c 100755 --- a/src/gausskernel/storage/access/transam/xlog.cpp +++ b/src/gausskernel/storage/access/transam/xlog.cpp @@ -9160,6 +9160,13 @@ void StartupXLOG(void) SSCLOGShmemClear(); SSMultiXactShmemClear(); } + ereport(LOG, (errmsg("[SS] Recovery instance %d, my instance %d, checkpoint record loc %X/%X, redo loc %X/%X", + g_instance.dms_cxt.SSRecoveryInfo.recovery_inst_id, + g_instance.attr.attr_storage.dms_attr.instance_id, + (uint32)(t_thrd.shemem_ptr_cxt.ControlFile->checkPoint >> 32), + (uint32)t_thrd.shemem_ptr_cxt.ControlFile->checkPoint, + (uint32)(t_thrd.shemem_ptr_cxt.ControlFile->checkPointCopy.redo >> 32), + (uint32)t_thrd.shemem_ptr_cxt.ControlFile->checkPointCopy.redo))); } else { xlogreader = XLogReaderAllocate(&XLogPageRead, &readprivate); } @@ -9611,6 +9618,8 @@ void StartupXLOG(void) t_thrd.xlog_cxt.InRecovery = false; } + g_instance.dms_cxt.SSRecoveryInfo.in_ondemand_recovery = false; + SetExtremeRtoMode(); if (SS_PRIMARY_MODE && ENABLE_ONDEMAND_RECOVERY && (SS_STANDBY_FAILOVER || SS_PRIMARY_NORMAL_REFORM) && t_thrd.xlog_cxt.InRecovery == true) { if (SSOndemandRecoveryExitNormal) { @@ -9622,8 +9631,6 @@ void StartupXLOG(void) SetOndemandExtremeRtoMode(); ereport(LOG, (errmsg("[On-demand] replayed in extreme rto ondemand recovery mode"))); } else { - g_instance.dms_cxt.SSRecoveryInfo.in_ondemand_recovery = false; - SetExtremeRtoMode(); ereport(LOG, (errmsg("[On-demand] do not allow replay in ondemand recovery if last ondemand recovery " "crash, replayed in extreme rto recovery mode"))); } From da42d9432a594f8989e3ca17b55cbff582a8b5cd Mon Sep 17 00:00:00 2001 From: luozihao <1165977584@qq.com> Date: Tue, 29 Aug 2023 19:26:55 +0800 Subject: [PATCH 173/304] =?UTF-8?q?whale=E6=8F=92=E4=BB=B6=E9=80=82?= =?UTF-8?q?=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- GNUmakefile.in | 1 + build/script/aarch64_opengauss_list | 3 +++ build/script/opengauss_release_list_ubuntu_single | 3 +++ build/script/x86_64_opengauss_list | 3 +++ src/common/backend/utils/fmgr/fmgr.cpp | 12 ++++++++++++ .../runtime/vecexecutor/vecexpression.cpp | 3 +++ src/include/knl/knl_instance.h | 1 + 7 files changed, 26 insertions(+) diff --git a/GNUmakefile.in b/GNUmakefile.in index 9a5919e38..b5b099d37 100644 --- a/GNUmakefile.in +++ b/GNUmakefile.in @@ -96,6 +96,7 @@ install: $(MAKE) -C contrib/dblink $@ $(MAKE) -C contrib/ndpplugin $@ @if test -d contrib/dolphin; then $(MAKE) -C contrib/dolphin $@; fi + @if test -d contrib/whale; then $(MAKE) -C contrib/whale $@; fi +@echo "openGauss installation complete." endif endif diff --git a/build/script/aarch64_opengauss_list b/build/script/aarch64_opengauss_list index 2904c867a..06d078ea2 100644 --- a/build/script/aarch64_opengauss_list +++ b/build/script/aarch64_opengauss_list @@ -79,6 +79,8 @@ ./share/postgresql/extension/dolphin--2.0--2.0.1.sql ./share/postgresql/extension/dolphin--1.1--2.0.sql ./share/postgresql/extension/dolphin--2.0--1.1.sql +./share/postgresql/extension/whale.control +./share/postgresql/extension/whale--1.0.sql ./share/postgresql/extension/dolphin--2.0.1--2.0.sql ./share/postgresql/extension/openGauss_expr_dolphin.ir ./share/postgresql/extension/assessment--1.0.sql @@ -773,6 +775,7 @@ ./lib/postgresql/proc_srclib ./lib/postgresql/security_plugin.so ./lib/postgresql/dolphin.so +./lib/postgresql/whale.so ./lib/postgresql/pg_upgrade_support.so ./lib/postgresql/java/pljava.jar ./lib/postgresql/postgres_fdw.so diff --git a/build/script/opengauss_release_list_ubuntu_single b/build/script/opengauss_release_list_ubuntu_single index dfd0958b4..4f3b8da5e 100644 --- a/build/script/opengauss_release_list_ubuntu_single +++ b/build/script/opengauss_release_list_ubuntu_single @@ -67,6 +67,8 @@ ./share/postgresql/extension/dolphin--2.0--1.0.sql ./share/postgresql/extension/dolphin--1.1--2.0.sql ./share/postgresql/extension/dolphin--2.0--1.1.sql +./share/postgresql/extension/whale.control +./share/postgresql/extension/whale--1.0.sql ./share/postgresql/extension/openGauss_expr_dolphin.ir ./share/postgresql/extension/file_fdw--1.0.sql ./share/postgresql/extension/plpgsql.control @@ -761,6 +763,7 @@ ./lib/postgresql/proc_srclib ./lib/postgresql/security_plugin.so ./lib/postgresql/dolphin.so +./lib/postgresql/whale.so ./lib/postgresql/pg_upgrade_support.so ./lib/postgresql/java/pljava.jar ./lib/postgresql/postgres_fdw.so diff --git a/build/script/x86_64_opengauss_list b/build/script/x86_64_opengauss_list index 4f1eee380..6bc6510f1 100644 --- a/build/script/x86_64_opengauss_list +++ b/build/script/x86_64_opengauss_list @@ -79,6 +79,8 @@ ./share/postgresql/extension/dolphin--2.0--2.0.1.sql ./share/postgresql/extension/dolphin--1.1--2.0.sql ./share/postgresql/extension/dolphin--2.0--1.1.sql +./share/postgresql/extension/whale.control +./share/postgresql/extension/whale--1.0.sql ./share/postgresql/extension/dolphin--2.0.1--2.0.sql ./share/postgresql/extension/openGauss_expr_dolphin.ir ./share/postgresql/extension/assessment--1.0.sql @@ -773,6 +775,7 @@ ./lib/postgresql/proc_srclib ./lib/postgresql/security_plugin.so ./lib/postgresql/dolphin.so +./lib/postgresql/whale.so ./lib/postgresql/pg_upgrade_support.so ./lib/postgresql/java/pljava.jar ./lib/postgresql/postgres_fdw.so diff --git a/src/common/backend/utils/fmgr/fmgr.cpp b/src/common/backend/utils/fmgr/fmgr.cpp index bd3538496..de6e9e2ce 100755 --- a/src/common/backend/utils/fmgr/fmgr.cpp +++ b/src/common/backend/utils/fmgr/fmgr.cpp @@ -157,6 +157,12 @@ static RegExternFunc plpgsql_function_table[] = { */ RegExternFunc b_plpgsql_function_table[3]; +/* + * Now for whale to rewrite plpgsql_call_handler, plpgsql_inline_handler + * and plpgsql_validator. + */ +RegExternFunc a_plpgsql_function_table[3]; + static HTAB* CFuncHash = NULL; static void fmgr_info_cxt_security(Oid functionId, FmgrInfo* finfo, MemoryContext mcxt, bool ignore_security); @@ -403,6 +409,12 @@ static PGFunction load_plpgsql_function(char* funcname) sizeof(b_plpgsql_function_table) / sizeof(b_plpgsql_function_table[0]), sizeof(RegExternFunc), ExternFuncComp); + } else if (u_sess->attr.attr_sql.whale) { + search_result = (RegExternFunc*)bsearch(&tmp_key, + a_plpgsql_function_table, + sizeof(a_plpgsql_function_table) / sizeof(a_plpgsql_function_table[0]), + sizeof(RegExternFunc), + ExternFuncComp); } if (search_result == NULL) { search_result = (RegExternFunc*)bsearch(&tmp_key, diff --git a/src/gausskernel/runtime/vecexecutor/vecexpression.cpp b/src/gausskernel/runtime/vecexecutor/vecexpression.cpp index 46fe14c89..624bf272c 100644 --- a/src/gausskernel/runtime/vecexecutor/vecexpression.cpp +++ b/src/gausskernel/runtime/vecexecutor/vecexpression.cpp @@ -1996,6 +1996,9 @@ void InitVectorFunction(FunctionCallInfo finfo, MemoryContext fcacheCxt) if (u_sess->attr.attr_sql.dolphin && g_instance.plugin_vec_func_cxt.vec_func_plugin[DOLPHIN_VEC] != NULL) { vec_func_hash = g_instance.plugin_vec_func_cxt.vec_func_plugin[DOLPHIN_VEC]; + } else if (u_sess->attr.attr_sql.whale && + g_instance.plugin_vec_func_cxt.vec_func_plugin[WHALE_VEC] != NULL) { + vec_func_hash = g_instance.plugin_vec_func_cxt.vec_func_plugin[WHALE_VEC]; } else #endif vec_func_hash = g_instance.vec_func_hash; diff --git a/src/include/knl/knl_instance.h b/src/include/knl/knl_instance.h index 1853d7035..b5ae35361 100755 --- a/src/include/knl/knl_instance.h +++ b/src/include/knl/knl_instance.h @@ -149,6 +149,7 @@ typedef struct knl_g_cost_context { enum plugin_vecfunc_type { DOLPHIN_VEC = 0, + WHALE_VEC, /* * This is the number of vecfunc hash tables. From 1cd2e5c26704aa735c8d3e2578450c10b6a951bb Mon Sep 17 00:00:00 2001 From: bowenliu Date: Tue, 29 Aug 2023 19:56:10 +0800 Subject: [PATCH 174/304] fix fatal when alter set err sstxnstatus size --- src/common/backend/utils/misc/guc/guc_storage.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/common/backend/utils/misc/guc/guc_storage.cpp b/src/common/backend/utils/misc/guc/guc_storage.cpp index ca3561f28..68de9bc48 100755 --- a/src/common/backend/utils/misc/guc/guc_storage.cpp +++ b/src/common/backend/utils/misc/guc/guc_storage.cpp @@ -6260,12 +6260,14 @@ static bool check_ss_txnstatus_cache_size(int* newval, void** extra, GucSource s const int minval = 8192; if (*newval < minval) { - ereport(FATAL, (errmsg("ss_txnstatus_cache_size set as %d, should be >8192 or 0.", *newval))); + ereport(ERROR, (errmsg("ss_txnstatus_cache_size set as %d, should be >8192 or 0.", *newval))); + return false; } if (*newval % NUM_TXNSTATUS_CACHE_PARTITIONS != 0) { - ereport(FATAL, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("ss_txnstatus_cache_size should be multiple of partition number 256."))); + return false; } return true; } From 345e5e849b64f08bfd5fa6d88b0c845501c9d383 Mon Sep 17 00:00:00 2001 From: jiangyan <18091841830@163.com> Date: Tue, 29 Aug 2023 20:56:33 +0800 Subject: [PATCH 175/304] =?UTF-8?q?=E5=B7=B2=E4=BF=AE=E6=94=B9expected/int?= =?UTF-8?q?8.out?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/test/regress/expected/int8.out | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/test/regress/expected/int8.out b/src/test/regress/expected/int8.out index 3f863c4d6..578379add 100644 --- a/src/test/regress/expected/int8.out +++ b/src/test/regress/expected/int8.out @@ -615,11 +615,11 @@ SELECT '' AS to_char_12, to_char(q2, 'FM9999999999999999.000') FROM INT8_TBL ORD SELECT '' AS to_char_13, to_char(q2, 'L9999999999999999.000') FROM INT8_TBL ORDER BY q2; to_char_13 | to_char ------------+------------------------ - | -4567890123456789.000 - | 123.000 - | 456.000 - | 4567890123456789.000 - | 4567890123456789.000 + | $-4567890123456789.000 + | $ 123.000 + | $ 456.000 + | $ 4567890123456789.000 + | $ 4567890123456789.000 | (6 rows) From 0621185867447c038bb9153e6ab8b251fff69d7f Mon Sep 17 00:00:00 2001 From: dongning12 Date: Tue, 29 Aug 2023 20:42:02 +0800 Subject: [PATCH 176/304] =?UTF-8?q?=E3=80=90=E8=B5=84=E6=BA=90=E6=B1=A0?= =?UTF-8?q?=E5=8C=96=E3=80=91=E5=8F=8A=E6=97=B6=E9=87=8A=E6=94=BExmin?= =?UTF-8?q?=E7=9A=84hash=E8=A1=A8=E7=9A=84=E6=9F=A5=E8=AF=A2seq=EF=BC=9B?= =?UTF-8?q?=E6=8E=A8=E8=BF=9BDMS=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/ddes/ddes_commit_id | 2 +- src/include/ddes/dms/dms_api.h | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/gausskernel/ddes/ddes_commit_id b/src/gausskernel/ddes/ddes_commit_id index 31dab8b27..a8bf771d9 100644 --- a/src/gausskernel/ddes/ddes_commit_id +++ b/src/gausskernel/ddes/ddes_commit_id @@ -1,3 +1,3 @@ -dms_commit_id=19301cb7b3dd7f959cf6269c6d2aa8c8f23804fa +dms_commit_id=d3b22f7482078513b98dcb3241e17b26962e8a36 dss_commit_id=d48ae4eaa803bb9dec414b25f26ba37fff8942df cbb_commit_id=41cadda4643656bcd856bd37d2b6908e44a5d46c \ No newline at end of file diff --git a/src/include/ddes/dms/dms_api.h b/src/include/ddes/dms/dms_api.h index 94524a99a..8aa775b9b 100644 --- a/src/include/ddes/dms/dms_api.h +++ b/src/include/ddes/dms/dms_api.h @@ -710,6 +710,7 @@ typedef unsigned int (*dms_calc_page_checksum)(void *db_handle, dms_buf_ctrl_t * typedef int (*dms_verify_page_checksum)(void *db_handle, dms_buf_ctrl_t *ctrl, unsigned int page_size, int cks); typedef int (*dms_update_node_oldest_xmin)(void *db_handle, unsigned char inst_id, unsigned long long oldest_xmin); typedef void (*dms_set_inst_behavior)(void *db_handle, dms_inst_behavior_t inst_behavior); +typedef int (*dms_db_prepare)(void *db_handle); typedef struct st_dms_callback { // used in reform @@ -854,6 +855,7 @@ typedef struct st_dms_callback { //for shared storage backup dms_set_inst_behavior set_inst_behavior; + dms_db_prepare db_prepare; } dms_callback_t; typedef struct st_dms_instance_net_addr { @@ -937,7 +939,7 @@ typedef enum st_protocol_version { #define DMS_LOCAL_MINOR_VER_WEIGHT 1000 #define DMS_LOCAL_MAJOR_VERSION 0 #define DMS_LOCAL_MINOR_VERSION 0 -#define DMS_LOCAL_VERSION 88 +#define DMS_LOCAL_VERSION 89 #ifdef __cplusplus } From 5376fde6fd5fe8e197af594ff5b4f4ce20460402 Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Tue, 29 Aug 2023 21:49:00 +0800 Subject: [PATCH 177/304] =?UTF-8?q?8.30=E5=9B=9E=E9=80=80=E6=8E=A8?= =?UTF-8?q?=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/ddes/ddes_commit_id | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gausskernel/ddes/ddes_commit_id b/src/gausskernel/ddes/ddes_commit_id index a8bf771d9..e2f8422e9 100644 --- a/src/gausskernel/ddes/ddes_commit_id +++ b/src/gausskernel/ddes/ddes_commit_id @@ -1,3 +1,3 @@ dms_commit_id=d3b22f7482078513b98dcb3241e17b26962e8a36 -dss_commit_id=d48ae4eaa803bb9dec414b25f26ba37fff8942df +dss_commit_id=45ae7916cbdda2b2e64c02811db30df565cf34fa cbb_commit_id=41cadda4643656bcd856bd37d2b6908e44a5d46c \ No newline at end of file From e90d82827903aa6803e2fcd996892e117cbd44a7 Mon Sep 17 00:00:00 2001 From: slbaiyi Date: Wed, 30 Aug 2023 09:56:36 +0800 Subject: [PATCH 178/304] commit --- src/bin/psql/common.cpp | 3 +++ src/bin/psql/mainloop.cpp | 15 ++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/bin/psql/common.cpp b/src/bin/psql/common.cpp index 3dd7604c4..39899e28e 100644 --- a/src/bin/psql/common.cpp +++ b/src/bin/psql/common.cpp @@ -51,6 +51,8 @@ static void set_searchpath_for_tmptbl(PGconn* conn); static bool AcceptResult(const PGresult* result, bool print_error = true); bool GetPrintResult(PGresult** results, bool is_explain, bool is_print, const char* query, bool print_error = true); +extern void resetCheckAfterReconn(); + /* Mutexes for child processes in parallel mode. * When a child gets a result successfully, it try to print the result to "pset.queryFout". * All the results should be printed by sequence, so a traffic light is needed. @@ -402,6 +404,7 @@ static bool CheckConnection(void) UnsyncVariables(); } else { fputs(_("Succeeded.\n"), stderr); + resetCheckAfterReconn(); } } diff --git a/src/bin/psql/mainloop.cpp b/src/bin/psql/mainloop.cpp index 73f59c468..798a810ca 100644 --- a/src/bin/psql/mainloop.cpp +++ b/src/bin/psql/mainloop.cpp @@ -101,7 +101,7 @@ static void SetSessionTimeout(const char* session_timeout) PQclear(StRes); } -static void JudgeEndStateInBFormat(const char* inputLine, bool &is_b_format, char* delimiter_name, bool is_new_lines) +static void JudgeEndStateInBFormat(const char* inputLine, bool &is_b_format, char* delimiter_name, bool is_new_lines, bool reset_check_after_reconn = false) { /* Convert inputLine to lowercase */ char *inputLine_temp = pg_strdup(inputLine); @@ -114,6 +114,13 @@ static void JudgeEndStateInBFormat(const char* inputLine, bool &is_b_format, cha char *tokenPtr = strstr(inputLine_temp, "delimiter"); char *tokenPtr1 = strstr(inputLine_temp, "\\c" ); errno_t rc = 0; + + /* reset the variables after reseting the losed connection to the server successfully */ + if (reset_check_after_reconn) { + is_just_one_check = false; + is_just_two_check = false; + return; + } if (!is_just_one_check) { res = PQexec(pset.db, "show sql_compatibility"); @@ -153,6 +160,12 @@ static void JudgeEndStateInBFormat(const char* inputLine, bool &is_b_format, cha inputLine_temp =NULL; } +void resetCheckAfterReconn() +{ + bool fake_param = false; + JudgeEndStateInBFormat("", fake_param, NULL, false, true); +} + static bool is_match_delimiter_name(const char* left, const char* right) { if (strlen(left) < strlen(right)) { From 2e41c8d4176f02b949c4a7affbf086846c117fe8 Mon Sep 17 00:00:00 2001 From: openGaussDev Date: Mon, 7 Aug 2023 20:09:36 +0800 Subject: [PATCH 179/304] dr switchover hadr_num check fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Offering: openGaussDev More detail: 可选 Match-id-574e112e6d668adeb6d802a4a773b48952149bb2 --- .../storage/access/transam/xlogfuncs.cpp | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/gausskernel/storage/access/transam/xlogfuncs.cpp b/src/gausskernel/storage/access/transam/xlogfuncs.cpp index 2636fcfae..9533951c6 100755 --- a/src/gausskernel/storage/access/transam/xlogfuncs.cpp +++ b/src/gausskernel/storage/access/transam/xlogfuncs.cpp @@ -2046,6 +2046,23 @@ Datum gs_streaming_dr_in_switchover(PG_FUNCTION_ARGS) Datum gs_streaming_dr_service_truncation_check(PG_FUNCTION_ARGS) { #ifndef ENABLE_LITE_MODE + int dr_sender_num = 0; + + for (int i = 1; i < MAX_REPLNODE_NUM; i++) { + ReplConnInfo *replConnInfo = NULL; + replConnInfo = t_thrd.postmaster_cxt.ReplConnArray[i]; + + /* Number of DR replconninfo */ + if (replConnInfo != NULL && replConnInfo->isCrossRegion) { + dr_sender_num++; + } + } + if (IS_PGXC_COORDINATOR) { + g_instance.streaming_dr_cxt.hadrWalSndNum = dr_sender_num; + } else { + g_instance.streaming_dr_cxt.hadrWalSndNum = dr_sender_num > 0 ? 1 : 0; + } + for (int i = 0; i < g_instance.attr.attr_storage.max_wal_senders; i++) { /* use volatile pointer to prevent code rearrangement */ volatile WalSnd *walsnd = &t_thrd.walsender_cxt.WalSndCtl->walsnds[i]; @@ -2057,7 +2074,6 @@ Datum gs_streaming_dr_service_truncation_check(PG_FUNCTION_ARGS) SpinLockAcquire(&walsnd->mutex); if (walsnd->interactiveState == SDRS_DEFAULT) { walsnd->interactiveState = SDRS_INTERACTION_BEGIN; - g_instance.streaming_dr_cxt.hadrWalSndNum++; } SpinLockRelease(&walsnd->mutex); } From 06aaf91dfc8943f3a0ad7a47b88352dff88b6096 Mon Sep 17 00:00:00 2001 From: openGaussDev Date: Sat, 26 Aug 2023 11:01:29 +0800 Subject: [PATCH 180/304] add new charset GB18030-2022 Offering: openGaussDev More detail:rename privategauss to sql_extension Match-id-60b5cadd18e5fcf82b97fecf7a06851e3e2739b2 --- src/bin/initdb/initdb.cpp | 3 +- src/common/backend/utils/init/globals.cpp | 3 +- .../backend/utils/mb/Unicode/CMakeLists.txt | 7 + src/common/backend/utils/mb/Unicode/Makefile | 4 + .../utils/mb/Unicode/UCS_to_GB18030_2022.pl | 92 ++ .../utils/mb/Unicode/gb-18030-2022.xml | 78 ++ .../utils/mb/Unicode/gb18030_to_utf8_2022.map | 40 + .../utils/mb/Unicode/utf8_to_gb18030_2022.map | 40 + src/common/backend/utils/mb/conv.cpp | 21 + .../utils/mb/conversion_procs/CMakeLists.txt | 1 + .../utils/mb/conversion_procs/Makefile | 2 + .../conversion_procs/conversion_create.sql.in | 2 + .../utf8_and_gb18030/utf8_and_gb18030.cpp | 34 + src/common/backend/utils/mb/encnames.cpp | 2 + src/common/backend/utils/mb/mbutils.cpp | 30 +- src/common/backend/utils/mb/wchar.cpp | 6 + src/common/backend/utils/misc/guc.cpp | 4 + .../optimizer/commands/dbcommands.cpp | 10 +- .../optimizer/commands/variable.cpp | 2 +- .../rollback-post_catalog_maindb_92_908.sql | 19 + .../rollback-post_catalog_otherdb_92_908.sql | 19 + .../upgrade-post_catalog_maindb_92_908.sql | 36 + .../upgrade-post_catalog_otherdb_92_908.sql | 36 + src/include/mb/pg_wchar.h | 3 +- src/include/miscadmin.h | 1 + src/test/regress/input/ts_gb18030_utf8.source | 414 ++++++ .../regress/output/ts_gb18030_utf8.source | 1142 +++++++++++++++++ src/test/regress/parallel_schedule0 | 1 + src/test/regress/sql/ts_gb18030_utf8.sql | 20 - 29 files changed, 2039 insertions(+), 33 deletions(-) create mode 100644 src/common/backend/utils/mb/Unicode/UCS_to_GB18030_2022.pl create mode 100644 src/common/backend/utils/mb/Unicode/gb-18030-2022.xml create mode 100644 src/common/backend/utils/mb/Unicode/gb18030_to_utf8_2022.map create mode 100644 src/common/backend/utils/mb/Unicode/utf8_to_gb18030_2022.map create mode 100644 src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback-post_catalog_maindb_92_908.sql create mode 100644 src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback-post_catalog_otherdb_92_908.sql create mode 100644 src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_908.sql create mode 100644 src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_908.sql create mode 100644 src/test/regress/input/ts_gb18030_utf8.source create mode 100644 src/test/regress/output/ts_gb18030_utf8.source delete mode 100644 src/test/regress/sql/ts_gb18030_utf8.sql diff --git a/src/bin/initdb/initdb.cpp b/src/bin/initdb/initdb.cpp index 7419849f7..75d5e6bcf 100644 --- a/src/bin/initdb/initdb.cpp +++ b/src/bin/initdb/initdb.cpp @@ -3594,7 +3594,8 @@ static bool check_locale_encoding(const char* locale_encoding, int user_enc) #ifdef WIN32 user_enc == PG_UTF8 || #endif - user_enc == PG_SQL_ASCII)) { + user_enc == PG_SQL_ASCII || + (user_enc == PG_GB18030_2022 && locale_enc == PG_GB18030))) { write_stderr(_("%s: encoding mismatch\n"), progname); write_stderr(_("The encoding you selected (%s) and the encoding that the\n" "selected locale uses (%s) do not match. This would lead to\n" diff --git a/src/common/backend/utils/init/globals.cpp b/src/common/backend/utils/init/globals.cpp index 99f82fd13..d7824bfa3 100644 --- a/src/common/backend/utils/init/globals.cpp +++ b/src/common/backend/utils/init/globals.cpp @@ -75,12 +75,13 @@ bool will_shutdown = false; * NEXT | 92899 | ? | ? * ********************************************/ -const uint32 GRAND_VERSION_NUM = 92907; +const uint32 GRAND_VERSION_NUM = 92908; /******************************************** * 2.VERSION NUM FOR EACH FEATURE * Please write indescending order. ********************************************/ +const uint32 GB18030_2022_VERSION_NUM = 92908; const uint32 PARAM_MARK_VERSION_NUM = 92907; const uint32 TIMESCALE_DB_VERSION_NUM = 92904; const uint32 MULTI_CHARSET_VERSION_NUM = 92903; diff --git a/src/common/backend/utils/mb/Unicode/CMakeLists.txt b/src/common/backend/utils/mb/Unicode/CMakeLists.txt index 924c40d2e..8e7f4b480 100644 --- a/src/common/backend/utils/mb/Unicode/CMakeLists.txt +++ b/src/common/backend/utils/mb/Unicode/CMakeLists.txt @@ -154,3 +154,10 @@ add_custom_command( DEPENDS UCS_to_BIG5.pl BIG5.TXT CP950.TXT COMMENT "Now Generating *.map" ) + +add_custom_command( + OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/gb18030_to_utf8_2022.map ${CMAKE_CURRENT_SOURCE_DIR}/utf8_to_gb18030_2022.map + COMMAND perl ${CMAKE_CURRENT_SOURCE_DIR}/UCS_to_GB18030_2022.pl + DEPENDS UCS_to_GB18030_2022.pl gb-18030-2022.xml + COMMENT "Now Generating *.map" +) \ No newline at end of file diff --git a/src/common/backend/utils/mb/Unicode/Makefile b/src/common/backend/utils/mb/Unicode/Makefile index a777c8e6e..8580b491e 100644 --- a/src/common/backend/utils/mb/Unicode/Makefile +++ b/src/common/backend/utils/mb/Unicode/Makefile @@ -50,6 +50,7 @@ SPECIALMAPS = euc_cn_to_utf8.map utf8_to_euc_cn.map \ euc_tw_to_utf8.map utf8_to_euc_tw.map \ sjis_to_utf8.map utf8_to_sjis.map \ gb18030_to_utf8.map utf8_to_gb18030.map \ + gb18030_to_utf8_2022.map utf8_to_gb18030_2022.map \ big5_to_utf8.map utf8_to_big5.map MAPS = $(GENERICMAPS) $(SPECIALMAPS) @@ -89,6 +90,9 @@ sjis_to_utf8.map utf8_to_sjis.map : CP932.TXT gb18030_to_utf8.map utf8_to_gb18030.map : gb-18030-2000.xml $(PERL) $(srcdir)/UCS_to_GB18030.pl +gb18030_to_utf8_2022.map utf8_to_gb18030_2022.map : gb-18030-2022.xml + $(PERL) $(srcdir)/UCS_to_GB18030_2022.pl + big5_to_utf8.map utf8_to_big5.map : BIG5.TXT CP950.TXT $(PERL) $(srcdir)/UCS_to_BIG5.pl diff --git a/src/common/backend/utils/mb/Unicode/UCS_to_GB18030_2022.pl b/src/common/backend/utils/mb/Unicode/UCS_to_GB18030_2022.pl new file mode 100644 index 000000000..ec61f20fb --- /dev/null +++ b/src/common/backend/utils/mb/Unicode/UCS_to_GB18030_2022.pl @@ -0,0 +1,92 @@ +#! /usr/bin/perl +# +# Copyright (c) 2007-2012, 2023, PostgreSQL Global Development Group +# +# src/backend/utils/mb/Unicode/UCS_to_GB18030_2022.pl +# +# Generate UTF-8 <--> GB18030-2022 code conversion tables from +# "gb-18030-2022.xml" +# +# The lines we care about in the source file look like +# +# where the "u" field is the Unicode code point in hex, +# and the "b" field is the hex byte sequence for GB18030 + +require "ucs2utf.pl"; + + +$change_file = "gb-18030-2022.xml"; + +open(CODE_TABLE, $change_file) || die("cannot open $change_file"); + +while () +{ + next if (! m/= 0x80 && $ucs_code >= 0x0080) + { + $utf_code = &ucs2utf($ucs_code); + if ($code_u{$utf_code} ne "") + { + printf STDERR "Warning: duplicate UTF8: %04x\n", $ucs_code; + next; + } + if ($code_c{$code_gb} ne "") + { + printf STDERR "Warning: duplicate GB18030: %08x\n", $code_gb; + next; + } + $code_u{$utf_code} = $code_gb; + $code_c{$code_gb} = $utf_code; + $number++; + } +} +close(CODE_TABLE); + +$change_map_file = "gb18030_to_utf8_2022.map"; +open(CHANGE_MAP, "> $change_map_file") || die("cannot open $change_map_file"); +print CHANGE_MAP "static pg_local_to_utf LUmapGB18030_2022[ $number ] = {\n"; + +$count = $number; +for $pos (sort { $a <=> $b } keys(%code_c)) +{ + $utf_code = $code_c{$pos}; + $count--; + if ($count == 0) + { + printf CHANGE_MAP " {0x%04x, 0x%04x}\n", $pos, $utf_code; + } + else + { + printf CHANGE_MAP " {0x%04x, 0x%04x},\n", $pos, $utf_code; + } +} + +print CHANGE_MAP "};\n"; +close(CHANGE_MAP); + +$change_map_file = "utf8_to_gb18030_2022.map"; +open(CHANGE_MAP, "> $change_map_file") || die("cannot open $change_map_file"); +print CHANGE_MAP "static pg_utf_to_local ULmapGB18030_2022[ $number ] = {\n"; + +$count = $number; +for $pos (sort { $a <=> $b } keys(%code_u)) +{ + $code_gb = $code_u{$pos}; + $count--; + if ($count == 0) + { + printf CHANGE_MAP " {0x%04x, 0x%04x}\n", $pos, $code_gb; + } + else + { + printf CHANGE_MAP " {0x%04x, 0x%04x},\n", $pos, $code_gb; + } +} + +print CHANGE_MAP "};\n"; +close(CHANGE_MAP); diff --git a/src/common/backend/utils/mb/Unicode/gb-18030-2022.xml b/src/common/backend/utils/mb/Unicode/gb-18030-2022.xml new file mode 100644 index 000000000..79ce3ffc4 --- /dev/null +++ b/src/common/backend/utils/mb/Unicode/gb-18030-2022.xml @@ -0,0 +1,78 @@ + + + + + + A list of character encodings in which the location of the GB/T 13000 code is changed compared to version 2000 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/common/backend/utils/mb/Unicode/gb18030_to_utf8_2022.map b/src/common/backend/utils/mb/Unicode/gb18030_to_utf8_2022.map new file mode 100644 index 000000000..275420c45 --- /dev/null +++ b/src/common/backend/utils/mb/Unicode/gb18030_to_utf8_2022.map @@ -0,0 +1,40 @@ +static pg_local_to_utf LUmapGB18030_2022[ 38 ] = { + {0xa6d9, 0xefb890}, + {0xa6da, 0xefb892}, + {0xa6db, 0xefb891}, + {0xa6dc, 0xefb893}, + {0xa6dd, 0xefb894}, + {0xa6de, 0xefb895}, + {0xa6df, 0xefb896}, + {0xa6ec, 0xefb897}, + {0xa6ed, 0xefb898}, + {0xa6f3, 0xefb899}, + {0xa8bc, 0xe1b8bf}, + {0xfe59, 0xe9beb4}, + {0xfe61, 0xe9beb5}, + {0xfe66, 0xe9beb6}, + {0xfe67, 0xe9beb7}, + {0xfe6d, 0xe9beb8}, + {0xfe7e, 0xe9beb9}, + {0xfe90, 0xe9beba}, + {0xfea0, 0xe9bebb}, + {0x8135f437, 0xee9f87}, + {0x82359037, 0xeea09e}, + {0x82359038, 0xeea0a6}, + {0x82359039, 0xeea0ab}, + {0x82359130, 0xeea0ac}, + {0x82359131, 0xeea0b2}, + {0x82359132, 0xeea183}, + {0x82359133, 0xeea194}, + {0x82359134, 0xeea1a4}, + {0x84318236, 0xee9e8d}, + {0x84318237, 0xee9e8f}, + {0x84318238, 0xee9e8e}, + {0x84318239, 0xee9e90}, + {0x84318330, 0xee9e91}, + {0x84318331, 0xee9e92}, + {0x84318332, 0xee9e93}, + {0x84318333, 0xee9e94}, + {0x84318334, 0xee9e95}, + {0x84318335, 0xee9e96} +}; diff --git a/src/common/backend/utils/mb/Unicode/utf8_to_gb18030_2022.map b/src/common/backend/utils/mb/Unicode/utf8_to_gb18030_2022.map new file mode 100644 index 000000000..9fafde791 --- /dev/null +++ b/src/common/backend/utils/mb/Unicode/utf8_to_gb18030_2022.map @@ -0,0 +1,40 @@ +static pg_utf_to_local ULmapGB18030_2022[ 38 ] = { + {0xe1b8bf, 0xa8bc}, + {0xe9beb4, 0xfe59}, + {0xe9beb5, 0xfe61}, + {0xe9beb6, 0xfe66}, + {0xe9beb7, 0xfe67}, + {0xe9beb8, 0xfe6d}, + {0xe9beb9, 0xfe7e}, + {0xe9beba, 0xfe90}, + {0xe9bebb, 0xfea0}, + {0xee9e8d, 0x84318236}, + {0xee9e8e, 0x84318238}, + {0xee9e8f, 0x84318237}, + {0xee9e90, 0x84318239}, + {0xee9e91, 0x84318330}, + {0xee9e92, 0x84318331}, + {0xee9e93, 0x84318332}, + {0xee9e94, 0x84318333}, + {0xee9e95, 0x84318334}, + {0xee9e96, 0x84318335}, + {0xee9f87, 0x8135f437}, + {0xeea09e, 0x82359037}, + {0xeea0a6, 0x82359038}, + {0xeea0ab, 0x82359039}, + {0xeea0ac, 0x82359130}, + {0xeea0b2, 0x82359131}, + {0xeea183, 0x82359132}, + {0xeea194, 0x82359133}, + {0xeea1a4, 0x82359134}, + {0xefb890, 0xa6d9}, + {0xefb891, 0xa6db}, + {0xefb892, 0xa6da}, + {0xefb893, 0xa6dc}, + {0xefb894, 0xa6dd}, + {0xefb895, 0xa6de}, + {0xefb896, 0xa6df}, + {0xefb897, 0xa6ec}, + {0xefb898, 0xa6ed}, + {0xefb899, 0xa6f3} +}; diff --git a/src/common/backend/utils/mb/conv.cpp b/src/common/backend/utils/mb/conv.cpp index c36ca8724..816185d41 100644 --- a/src/common/backend/utils/mb/conv.cpp +++ b/src/common/backend/utils/mb/conv.cpp @@ -14,6 +14,8 @@ #include "postgres.h" #include "knl/knl_variable.h" #include "mb/pg_wchar.h" +#include "Unicode/gb18030_to_utf8_2022.map" +#include "Unicode/utf8_to_gb18030_2022.map" /* * LATINn ---> MIC when the charset's local codes map directly to MIC @@ -479,6 +481,16 @@ void UtfToLocal(const unsigned char *utf, int len, unsigned char *iso, const pg_ l = l_save; } /* Now check ordinary map */ + // add gb18030-2022 conv judge. + if (encoding == PG_GB18030_2022) { + p = (pg_utf_to_local*)bsearch(&iutf, ULmapGB18030_2022, + lengthof(ULmapGB18030_2022), sizeof(pg_utf_to_local), compare1); + if (p != NULL) { + iso = store_coded_char(iso, p->code); + continue; + } + } + p = (pg_utf_to_local *)bsearch(&iutf, map, mapsize, sizeof(pg_utf_to_local), compare1); if (p != NULL) { iso = store_coded_char(iso, p->code); @@ -602,6 +614,15 @@ void LocalToUtf(const unsigned char *iso, int len, unsigned char *utf, const pg_ iiso |= *iso++; } + // add gb18030-2022 conv judge + if (encoding == PG_GB18030_2022) { + p = (pg_local_to_utf*)bsearch(&iiso, LUmapGB18030_2022, + lengthof(LUmapGB18030_2022), sizeof(pg_local_to_utf), compare2); + if (p != NULL) { + utf = store_coded_char(utf, p->utf); + continue; + } + } p = (pg_local_to_utf*)bsearch(&iiso, map, mapsize, sizeof(pg_local_to_utf), compare2); if (p != NULL) { utf = store_coded_char(utf, p->utf); diff --git a/src/common/backend/utils/mb/conversion_procs/CMakeLists.txt b/src/common/backend/utils/mb/conversion_procs/CMakeLists.txt index 47def6cc9..25148ad9e 100755 --- a/src/common/backend/utils/mb/conversion_procs/CMakeLists.txt +++ b/src/common/backend/utils/mb/conversion_procs/CMakeLists.txt @@ -8,6 +8,7 @@ SET(unicode_cmd_src "${PROJECT_SRC_DIR}/common/backend/utils/mb/Unicode|||perl UCS_to_SJIS.pl|" "${PROJECT_SRC_DIR}/common/backend/utils/mb/Unicode|||perl UCS_to_GB18030.pl|" "${PROJECT_SRC_DIR}/common/backend/utils/mb/Unicode|||perl UCS_to_BIG5.pl|" + "${PROJECT_SRC_DIR}/common/backend/utils/mb/Unicode|||perl UCS_to_GB18030_2022.pl|" ) add_cmd_gen_when_configure(perl_target unicode_cmd_src) diff --git a/src/common/backend/utils/mb/conversion_procs/Makefile b/src/common/backend/utils/mb/conversion_procs/Makefile index c3b697d47..06937885e 100644 --- a/src/common/backend/utils/mb/conversion_procs/Makefile +++ b/src/common/backend/utils/mb/conversion_procs/Makefile @@ -120,6 +120,8 @@ CONVERSIONS = \ utf8_to_euc_tw UTF8 EUC_TW utf8_to_euc_tw utf8_and_euc_tw \ gb18030_to_utf8 GB18030 UTF8 gb18030_to_utf8 utf8_and_gb18030 \ utf8_to_gb18030 UTF8 GB18030 utf8_to_gb18030 utf8_and_gb18030 \ + gb18030_2022_to_utf8 GB18030_2022 UTF8 gb18030_2022_to_utf8 utf8_and_gb18030 \ + utf8_to_gb18030_2022 UTF8 GB18030_2022 utf8_to_gb18030_2022 utf8_and_gb18030 \ gbk_to_utf8 GBK UTF8 gbk_to_utf8 utf8_and_gbk \ utf8_to_gbk UTF8 GBK utf8_to_gbk utf8_and_gbk \ utf8_to_iso_8859_2 UTF8 LATIN2 utf8_to_iso8859 utf8_and_iso8859 \ diff --git a/src/common/backend/utils/mb/conversion_procs/conversion_create.sql.in b/src/common/backend/utils/mb/conversion_procs/conversion_create.sql.in index 9fa3be54b..899b68080 100644 --- a/src/common/backend/utils/mb/conversion_procs/conversion_create.sql.in +++ b/src/common/backend/utils/mb/conversion_procs/conversion_create.sql.in @@ -130,3 +130,5 @@ shift_jis_2004_to_utf8 SHIFT_JIS_2004 UTF8 shift_jis_2004_to_utf8 utf8_and_sjis2 utf8_to_shift_jis_2004 UTF8 SHIFT_JIS_2004 utf8_to_shift_jis_2004 utf8_and_sjis2004 euc_jis_2004_to_shift_jis_2004 EUC_JIS_2004 SHIFT_JIS_2004 euc_jis_2004_to_shift_jis_2004 euc2004_sjis2004 shift_jis_2004_to_euc_jis_2004 SHIFT_JIS_2004 EUC_JIS_2004 shift_jis_2004_to_euc_jis_2004 euc2004_sjis2004 +gb18030_2022_to_utf8 GB18030_2022 UTF8 gb18030_2022_to_utf8 utf8_and_gb18030 +utf8_to_gb18030_2022 UTF8 GB18030_2022 utf8_to_gb18030_2022 utf8_and_gb18030 \ No newline at end of file diff --git a/src/common/backend/utils/mb/conversion_procs/utf8_and_gb18030/utf8_and_gb18030.cpp b/src/common/backend/utils/mb/conversion_procs/utf8_and_gb18030/utf8_and_gb18030.cpp index 52277e6c9..ff3c43eef 100644 --- a/src/common/backend/utils/mb/conversion_procs/utf8_and_gb18030/utf8_and_gb18030.cpp +++ b/src/common/backend/utils/mb/conversion_procs/utf8_and_gb18030/utf8_and_gb18030.cpp @@ -23,9 +23,15 @@ PG_MODULE_MAGIC; PG_FUNCTION_INFO_V1(gb18030_to_utf8); PG_FUNCTION_INFO_V1(utf8_to_gb18030); +PG_FUNCTION_INFO_V1(gb18030_2022_to_utf8); +PG_FUNCTION_INFO_V1(utf8_to_gb18030_2022); + extern "C" Datum gb18030_to_utf8(PG_FUNCTION_ARGS); extern "C" Datum utf8_to_gb18030(PG_FUNCTION_ARGS); +extern "C" Datum gb18030_2022_to_utf8(PG_FUNCTION_ARGS); +extern "C" Datum utf8_to_gb18030_2022(PG_FUNCTION_ARGS); + /* * Convert 4-byte GB18030 characters to and from a linear code space * @@ -195,3 +201,31 @@ Datum utf8_to_gb18030(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } + +// convert the GB18030-2022 code to the UTF8 code +Datum gb18030_2022_to_utf8(PG_FUNCTION_ARGS) +{ + unsigned char* src = (unsigned char*)PG_GETARG_CSTRING(2); + unsigned char* dest = (unsigned char*)PG_GETARG_CSTRING(3); + int len = PG_GETARG_INT32(4); + // check whether the conversion relationship between two character sets exists. + CHECK_ENCODING_CONVERSION_ARGS(PG_GB18030_2022, PG_UTF8); + + LocalToUtf(src, len, dest, LUmapGB18030, lengthof(LUmapGB18030), NULL, 0, conv_18030_to_utf8, PG_GB18030_2022); + + PG_RETURN_VOID(); +} + +// convert the UTF8 code to the GB18030-2022 code. +Datum utf8_to_gb18030_2022(PG_FUNCTION_ARGS) +{ + unsigned char* src = (unsigned char*)PG_GETARG_CSTRING(2); + unsigned char* dest = (unsigned char*)PG_GETARG_CSTRING(3); + int len = PG_GETARG_INT32(4); + // check whether the conversion relationship between two character sets exists. + CHECK_ENCODING_CONVERSION_ARGS(PG_UTF8, PG_GB18030_2022); + + UtfToLocal(src, len, dest, ULmapGB18030, lengthof(ULmapGB18030), NULL, 0, conv_utf8_to_18030, PG_GB18030_2022); + + PG_RETURN_VOID(); +} \ No newline at end of file diff --git a/src/common/backend/utils/mb/encnames.cpp b/src/common/backend/utils/mb/encnames.cpp index 9481606e6..4cda548b2 100644 --- a/src/common/backend/utils/mb/encnames.cpp +++ b/src/common/backend/utils/mb/encnames.cpp @@ -45,6 +45,7 @@ pg_encname pg_encname_tbl[] = { {"euckr", PG_EUC_KR}, /* EUC-KR; Extended Unix Code for Korean, KS X 1001 standard */ {"euctw", PG_EUC_TW}, /* EUC-TW; Extended Unix Code for traditional Chinese */ {"gb18030", PG_GB18030}, /* GB18030;GB18030 */ + {"gb180302022", PG_GB18030_2022}, /* GB18030-2022;version 2022 for GB18030 */ {"gbk", PG_GBK}, /* GBK; Chinese Windows CodePage 936 simplified Chinese */ {"iso88591", PG_LATIN1}, /* ISO-8859-1; RFC1345,KXS2 */ {"iso885910", PG_LATIN6}, /* ISO-8859-10; RFC1345,KXS2 */ @@ -188,6 +189,7 @@ pg_enc2name pg_enc2name_tbl[] = {DEF_ENC2NAME(SQL_ASCII, 0), DEF_ENC2NAME(WIN1257, 1257), DEF_ENC2NAME(KOI8U, 21866), DEF_ENC2NAME(GB18030, 54936), + DEF_ENC2NAME(GB18030_2022, 54936), DEF_ENC2NAME(SJIS, 932), DEF_ENC2NAME(BIG5, 950), DEF_ENC2NAME(UHC, 0), diff --git a/src/common/backend/utils/mb/mbutils.cpp b/src/common/backend/utils/mb/mbutils.cpp index e336cea1d..85663c196 100644 --- a/src/common/backend/utils/mb/mbutils.cpp +++ b/src/common/backend/utils/mb/mbutils.cpp @@ -39,6 +39,24 @@ typedef struct ConvProcInfo { static char* perform_default_encoding_conversion(const char* src, int len, bool is_client_to_server); static int cliplen(const char* str, int len, int limit); +// Determine whether the current case needs to be converted +bool NoNeedToConvert(int srcEncoding, int destEncoding) +{ + if (srcEncoding == destEncoding) { + return true; + } + if (srcEncoding == PG_SQL_ASCII || destEncoding == PG_SQL_ASCII) { + return true; + } + if (srcEncoding == PG_GB18030_2022 && destEncoding == PG_GB18030) { + return true; + } + if (srcEncoding == PG_GB18030 && destEncoding == PG_GB18030_2022) { + return true; + } + return false; +} + /* * Prepare for a future call to SetClientEncoding. Success should mean * that SetClientEncoding is guaranteed to succeed for this encoding request. @@ -66,7 +84,7 @@ int PrepareClientEncoding(int encoding) * Check for cases that require no conversion function. */ current_server_encoding = GetDatabaseEncoding(); - if (current_server_encoding == encoding || current_server_encoding == PG_SQL_ASCII || encoding == PG_SQL_ASCII) { + if (NoNeedToConvert(current_server_encoding, encoding)) { return 0; } @@ -159,7 +177,7 @@ int SetClientEncoding(int encoding) * Check for cases that require no conversion function. */ current_server_encoding = GetDatabaseEncoding(); - if (current_server_encoding == encoding || current_server_encoding == PG_SQL_ASCII || encoding == PG_SQL_ASCII) { + if (NoNeedToConvert(current_server_encoding, encoding)) { u_sess->mb_cxt.ClientEncoding = &pg_enc2name_tbl[encoding]; u_sess->mb_cxt.ToServerConvProc = NULL; u_sess->mb_cxt.ToClientConvProc = NULL; @@ -277,10 +295,7 @@ unsigned char* pg_do_encoding_conversion(unsigned char* src, int len, int src_en if (!IsTransactionState()) { return src; } - if (src_encoding == dest_encoding) { - return src; - } - if (src_encoding == PG_SQL_ASCII || dest_encoding == PG_SQL_ASCII) { + if (NoNeedToConvert(src_encoding, dest_encoding)) { return src; } if (len <= 0) { @@ -673,7 +688,8 @@ char* pg_any_to_server(const char* s, int len, int encoding) bulkload_illegal_chars_conversion = true; } - if (encoding == u_sess->mb_cxt.DatabaseEncoding->encoding || encoding == PG_SQL_ASCII) { + if (encoding == u_sess->mb_cxt.DatabaseEncoding->encoding || encoding == PG_SQL_ASCII || + (encoding == PG_GB18030 && u_sess->mb_cxt.DatabaseEncoding->encoding == PG_GB18030_2022)) { /* * No conversion is needed, but we must still validate the data. */ diff --git a/src/common/backend/utils/mb/wchar.cpp b/src/common/backend/utils/mb/wchar.cpp index be064bf2c..3d95322a0 100644 --- a/src/common/backend/utils/mb/wchar.cpp +++ b/src/common/backend/utils/mb/wchar.cpp @@ -1990,6 +1990,12 @@ pg_wchar_tbl pg_wchar_table[] = { pg_gb18030_dsplen, pg_gb18030_verifier, 4}, /* PG_GB18030 */ + {pg_gb180302wchar_with_len, + pg_wchar2gb18030_with_len, + pg_gb18030_mblen, + pg_gb18030_dsplen, + pg_gb18030_verifier, + 4}, /* PG_GB18030_2022 */ {0, 0, pg_sjis_mblen, pg_sjis_dsplen, pg_sjis_verifier, 2}, /* PG_SJIS */ {0, 0, pg_big5_mblen, pg_big5_dsplen, pg_big5_verifier, 2}, /* PG_BIG5 */ {0, 0, pg_uhc_mblen, pg_uhc_dsplen, pg_uhc_verifier, 2}, /* PG_UHC */ diff --git a/src/common/backend/utils/misc/guc.cpp b/src/common/backend/utils/misc/guc.cpp index 87f65f653..8c382b376 100755 --- a/src/common/backend/utils/misc/guc.cpp +++ b/src/common/backend/utils/misc/guc.cpp @@ -174,6 +174,7 @@ #include "utils/guc_resource.h" #include "utils/mem_snapshot.h" #include "nodes/parsenodes_common.h" +#include "mb/pg_wchar.h" #ifndef PG_KRB_SRVTAB #define PG_KRB_SRVTAB "" @@ -8306,6 +8307,9 @@ static void set_config_sourcefile(const char* name, char* sourcefile, int source */ void SetConfigOption(const char* name, const char* value, GucContext context, GucSource source) { + if (strcmp(name, "client_encoding") == 0 && pg_char_to_encoding(value) == PG_GB18030_2022) { + value = "gb18030"; + } (void)set_config_option(name, value, context, source, GUC_ACTION_SET, true, 0); } diff --git a/src/gausskernel/optimizer/commands/dbcommands.cpp b/src/gausskernel/optimizer/commands/dbcommands.cpp index df588ce4d..89f68b399 100644 --- a/src/gausskernel/optimizer/commands/dbcommands.cpp +++ b/src/gausskernel/optimizer/commands/dbcommands.cpp @@ -267,6 +267,10 @@ Oid createdb(const CreatedbStmt* stmt) if (encoding < 0) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("%s is not a valid encoding name", encoding_name))); + if (t_thrd.proc->workingVersionNum < GB18030_2022_VERSION_NUM && encoding == PG_GB18030_2022) { + ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("Not support to create database encoding %s in upgrade!", encoding_name))); + } } else ereport(ERROR, (errcode(ERRCODE_UNRECOGNIZED_NODE_TYPE), @@ -783,7 +787,8 @@ void check_encoding_locale_matches(int encoding, const char* collate, const char #ifdef WIN32 encoding == PG_UTF8 || #endif - (encoding == PG_SQL_ASCII && superuser()))) + (encoding == PG_SQL_ASCII && superuser() || + (encoding == PG_GB18030_2022 && ctype_encoding == PG_GB18030)))) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("encoding \"%s\" does not match locale \"%s\"", pg_encoding_to_char(encoding), ctype), @@ -794,7 +799,8 @@ void check_encoding_locale_matches(int encoding, const char* collate, const char #ifdef WIN32 encoding == PG_UTF8 || #endif - (encoding == PG_SQL_ASCII && superuser()))) + (encoding == PG_SQL_ASCII && superuser() || + (encoding == PG_GB18030_2022 && collate_encoding == PG_GB18030)))) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("encoding \"%s\" does not match locale \"%s\"", pg_encoding_to_char(encoding), collate), diff --git a/src/gausskernel/optimizer/commands/variable.cpp b/src/gausskernel/optimizer/commands/variable.cpp index 5f5acd1d7..bd1cb4a99 100644 --- a/src/gausskernel/optimizer/commands/variable.cpp +++ b/src/gausskernel/optimizer/commands/variable.cpp @@ -762,7 +762,7 @@ bool check_client_encoding(char** newval, void** extra, GucSource source) /* Look up the encoding by name */ encoding = pg_valid_client_encoding(*newval); - if (encoding < 0) { + if (encoding < 0 || encoding == PG_GB18030_2022) { return false; } diff --git a/src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback-post_catalog_maindb_92_908.sql b/src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback-post_catalog_maindb_92_908.sql new file mode 100644 index 000000000..f1bdb839d --- /dev/null +++ b/src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback-post_catalog_maindb_92_908.sql @@ -0,0 +1,19 @@ +-- ---------------------------------------------------------------- +-- roolback pg_catalog.pg_conversion +-- ---------------------------------------------------------------- + +delete from pg_catalog.pg_conversion where conname = 'gb18030_2022_to_utf8'; +delete from pg_catalog.pg_conversion where conname = 'utf8_to_gb18030_2022'; +DROP FUNCTION IF EXISTS pg_catalog.gb18030_2022_to_utf8(integer, integer, cstring, internal, integer) CASCADE; +DROP FUNCTION IF EXISTS pg_catalog.utf8_to_gb18030_2022(integer, integer, cstring, internal, integer) CASCADE; + +UPDATE pg_catalog.pg_conversion SET conforencoding=37 WHERE conname like 'sjis_to_%'; +UPDATE pg_catalog.pg_conversion SET contoencoding=37 WHERE conname like '%_to_sjis'; +UPDATE pg_catalog.pg_conversion SET conforencoding=38 WHERE conname like 'big5_to_%'; +UPDATE pg_catalog.pg_conversion SET contoencoding=38 WHERE conname like '%_to_big5'; +UPDATE pg_catalog.pg_conversion SET conforencoding=39 WHERE conname like 'uhc_to_%'; +UPDATE pg_catalog.pg_conversion SET contoencoding=39 WHERE conname like '%_to_uhc'; +UPDATE pg_catalog.pg_conversion SET conforencoding=40 WHERE conname like 'johab_to_%'; +UPDATE pg_catalog.pg_conversion SET contoencoding=40 WHERE conname like '%to_johab'; +UPDATE pg_catalog.pg_conversion SET conforencoding=41 WHERE conname like 'shift_jis_2004_to_%'; +UPDATE pg_catalog.pg_conversion SET contoencoding=41 WHERE conname like '%to_shift_jis_2004'; \ No newline at end of file diff --git a/src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback-post_catalog_otherdb_92_908.sql b/src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback-post_catalog_otherdb_92_908.sql new file mode 100644 index 000000000..f1bdb839d --- /dev/null +++ b/src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback-post_catalog_otherdb_92_908.sql @@ -0,0 +1,19 @@ +-- ---------------------------------------------------------------- +-- roolback pg_catalog.pg_conversion +-- ---------------------------------------------------------------- + +delete from pg_catalog.pg_conversion where conname = 'gb18030_2022_to_utf8'; +delete from pg_catalog.pg_conversion where conname = 'utf8_to_gb18030_2022'; +DROP FUNCTION IF EXISTS pg_catalog.gb18030_2022_to_utf8(integer, integer, cstring, internal, integer) CASCADE; +DROP FUNCTION IF EXISTS pg_catalog.utf8_to_gb18030_2022(integer, integer, cstring, internal, integer) CASCADE; + +UPDATE pg_catalog.pg_conversion SET conforencoding=37 WHERE conname like 'sjis_to_%'; +UPDATE pg_catalog.pg_conversion SET contoencoding=37 WHERE conname like '%_to_sjis'; +UPDATE pg_catalog.pg_conversion SET conforencoding=38 WHERE conname like 'big5_to_%'; +UPDATE pg_catalog.pg_conversion SET contoencoding=38 WHERE conname like '%_to_big5'; +UPDATE pg_catalog.pg_conversion SET conforencoding=39 WHERE conname like 'uhc_to_%'; +UPDATE pg_catalog.pg_conversion SET contoencoding=39 WHERE conname like '%_to_uhc'; +UPDATE pg_catalog.pg_conversion SET conforencoding=40 WHERE conname like 'johab_to_%'; +UPDATE pg_catalog.pg_conversion SET contoencoding=40 WHERE conname like '%to_johab'; +UPDATE pg_catalog.pg_conversion SET conforencoding=41 WHERE conname like 'shift_jis_2004_to_%'; +UPDATE pg_catalog.pg_conversion SET contoencoding=41 WHERE conname like '%to_shift_jis_2004'; \ No newline at end of file diff --git a/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_908.sql b/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_908.sql new file mode 100644 index 000000000..109cf32fc --- /dev/null +++ b/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_908.sql @@ -0,0 +1,36 @@ +-- ---------------------------------------------------------------- +-- upgrade pg_catalog.pg_conversion +-- ---------------------------------------------------------------- + +DROP FUNCTION IF EXISTS pg_catalog.gb18030_2022_to_utf8(integer, integer, cstring, internal, integer) CASCADE; +CREATE OR REPLACE FUNCTION pg_catalog.gb18030_2022_to_utf8(integer, integer, cstring, internal, integer) +RETURNS void +LANGUAGE c +STRICT NOT FENCED NOT SHIPPABLE +AS '$libdir/utf8_and_gb18030', $function$gb18030_2022_to_utf8$function$; +COMMENT ON FUNCTION pg_catalog.gb18030_2022_to_utf8(INTEGER, INTEGER, CSTRING, INTERNAL, INTEGER) +IS 'internal conversion function for GB18030_2022 to UTF8'; + +insert into pg_catalog.pg_conversion values ('gb18030_2022_to_utf8', 11, 10, 37, 7, 'gb18030_2022_to_utf8', true); + +DROP FUNCTION IF EXISTS pg_catalog.utf8_to_gb18030_2022(integer, integer, cstring, internal, integer) CASCADE; +CREATE OR REPLACE FUNCTION pg_catalog.utf8_to_gb18030_2022(integer, integer, cstring, internal, integer) +RETURNS void +LANGUAGE c +STRICT NOT FENCED NOT SHIPPABLE +AS '$libdir/utf8_and_gb18030', $function$utf8_to_gb18030_2022$function$; +COMMENT ON FUNCTION pg_catalog.utf8_to_gb18030_2022(INTEGER, INTEGER, CSTRING, INTERNAL, INTEGER) +IS 'internal conversion function for UTF8 to GB18030_2022'; + +insert into pg_catalog.pg_conversion values ('utf8_to_gb18030_2022', 11, 10, 7, 37, 'utf8_to_gb18030_2022', true); + +UPDATE pg_catalog.pg_conversion SET conforencoding=38 WHERE conname like 'sjis_to_%'; +UPDATE pg_catalog.pg_conversion SET contoencoding=38 WHERE conname like '%_to_sjis'; +UPDATE pg_catalog.pg_conversion SET conforencoding=39 WHERE conname like 'big5_to_%'; +UPDATE pg_catalog.pg_conversion SET contoencoding=39 WHERE conname like '%_to_big5'; +UPDATE pg_catalog.pg_conversion SET conforencoding=40 WHERE conname like 'uhc_to_%'; +UPDATE pg_catalog.pg_conversion SET contoencoding=40 WHERE conname like '%_to_uhc'; +UPDATE pg_catalog.pg_conversion SET conforencoding=41 WHERE conname like 'johab_to_%'; +UPDATE pg_catalog.pg_conversion SET contoencoding=41 WHERE conname like '%to_johab'; +UPDATE pg_catalog.pg_conversion SET conforencoding=42 WHERE conname like 'shift_jis_2004_to_%'; +UPDATE pg_catalog.pg_conversion SET contoencoding=42 WHERE conname like '%to_shift_jis_2004'; \ No newline at end of file diff --git a/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_908.sql b/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_908.sql new file mode 100644 index 000000000..109cf32fc --- /dev/null +++ b/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_908.sql @@ -0,0 +1,36 @@ +-- ---------------------------------------------------------------- +-- upgrade pg_catalog.pg_conversion +-- ---------------------------------------------------------------- + +DROP FUNCTION IF EXISTS pg_catalog.gb18030_2022_to_utf8(integer, integer, cstring, internal, integer) CASCADE; +CREATE OR REPLACE FUNCTION pg_catalog.gb18030_2022_to_utf8(integer, integer, cstring, internal, integer) +RETURNS void +LANGUAGE c +STRICT NOT FENCED NOT SHIPPABLE +AS '$libdir/utf8_and_gb18030', $function$gb18030_2022_to_utf8$function$; +COMMENT ON FUNCTION pg_catalog.gb18030_2022_to_utf8(INTEGER, INTEGER, CSTRING, INTERNAL, INTEGER) +IS 'internal conversion function for GB18030_2022 to UTF8'; + +insert into pg_catalog.pg_conversion values ('gb18030_2022_to_utf8', 11, 10, 37, 7, 'gb18030_2022_to_utf8', true); + +DROP FUNCTION IF EXISTS pg_catalog.utf8_to_gb18030_2022(integer, integer, cstring, internal, integer) CASCADE; +CREATE OR REPLACE FUNCTION pg_catalog.utf8_to_gb18030_2022(integer, integer, cstring, internal, integer) +RETURNS void +LANGUAGE c +STRICT NOT FENCED NOT SHIPPABLE +AS '$libdir/utf8_and_gb18030', $function$utf8_to_gb18030_2022$function$; +COMMENT ON FUNCTION pg_catalog.utf8_to_gb18030_2022(INTEGER, INTEGER, CSTRING, INTERNAL, INTEGER) +IS 'internal conversion function for UTF8 to GB18030_2022'; + +insert into pg_catalog.pg_conversion values ('utf8_to_gb18030_2022', 11, 10, 7, 37, 'utf8_to_gb18030_2022', true); + +UPDATE pg_catalog.pg_conversion SET conforencoding=38 WHERE conname like 'sjis_to_%'; +UPDATE pg_catalog.pg_conversion SET contoencoding=38 WHERE conname like '%_to_sjis'; +UPDATE pg_catalog.pg_conversion SET conforencoding=39 WHERE conname like 'big5_to_%'; +UPDATE pg_catalog.pg_conversion SET contoencoding=39 WHERE conname like '%_to_big5'; +UPDATE pg_catalog.pg_conversion SET conforencoding=40 WHERE conname like 'uhc_to_%'; +UPDATE pg_catalog.pg_conversion SET contoencoding=40 WHERE conname like '%_to_uhc'; +UPDATE pg_catalog.pg_conversion SET conforencoding=41 WHERE conname like 'johab_to_%'; +UPDATE pg_catalog.pg_conversion SET contoencoding=41 WHERE conname like '%to_johab'; +UPDATE pg_catalog.pg_conversion SET conforencoding=42 WHERE conname like 'shift_jis_2004_to_%'; +UPDATE pg_catalog.pg_conversion SET contoencoding=42 WHERE conname like '%to_shift_jis_2004'; \ No newline at end of file diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h index dadc6b8f9..22c553a2c 100644 --- a/src/include/mb/pg_wchar.h +++ b/src/include/mb/pg_wchar.h @@ -223,6 +223,7 @@ typedef enum pg_enc { PG_WIN1257, /* windows-1257 */ PG_KOI8U, /* KOI8-U */ PG_GB18030, /* GB18030 */ + PG_GB18030_2022, /* GB18030-2022 */ /* PG_ENCODING_BE_LAST points to the above entry */ /* followings are for client encoding only */ @@ -235,7 +236,7 @@ typedef enum pg_enc { } pg_enc; -#define PG_ENCODING_BE_LAST PG_GB18030 +#define PG_ENCODING_BE_LAST PG_GB18030_2022 /* * Please use these tests before access to pg_encconv_tbl[] diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h index 056ebc32a..e55ceaede 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h @@ -135,6 +135,7 @@ extern const uint32 CREATE_INDEX_IF_NOT_EXISTS_VERSION_NUM; extern const uint32 SLOW_SQL_VERSION_NUM; extern const uint32 INDEX_HINT_VERSION_NUM; extern const uint32 CREATE_TABLE_AS_VERSION_NUM; +extern const uint32 GB18030_2022_VERSION_NUM; extern void register_backend_version(uint32 backend_version); extern bool contain_backend_version(uint32 version_number); diff --git a/src/test/regress/input/ts_gb18030_utf8.source b/src/test/regress/input/ts_gb18030_utf8.source new file mode 100644 index 000000000..d24e74de0 --- /dev/null +++ b/src/test/regress/input/ts_gb18030_utf8.source @@ -0,0 +1,414 @@ +create database gb18030_2022 encoding='gb18030-2022' LC_COLLATE='zh_CN.GB18030' LC_CTYPE ='zh_CN.GB18030' TEMPLATE=template0; +\c gb18030_2022 + +show server_encoding; +set client_encoding = 'UTF8'; +show client_encoding; +--1. 编码映射关系发生改变的字符： +-- 查看未发生变化的字符串： +select convert_to('中国', 'GB18030-2022'); + +--插入了涉及GB18030-2000升级GB18030-2022后GB18030与UTF-8转换关系变更的19个字符。 +create table tb_test(id int, content text); + +insert into tb_test (id , content) +select 1, convert_from('\xA8BC', 'GB18030-2022'); + +insert into tb_test (id , content) +select 2, convert_from('\xA6D9', 'GB18030-2022'); + +insert into tb_test (id , content) +select 3, convert_from('\xA6DA', 'GB18030-2022'); + +insert into tb_test (id , content) +select 4, convert_from('\xA6DB', 'GB18030-2022'); + +insert into tb_test (id , content) +select 5, convert_from('\xA6DC', 'GB18030-2022'); + +insert into tb_test (id , content) +select 6, convert_from('\xA6DD', 'GB18030-2022'); + +insert into tb_test (id , content) +select 7, convert_from('\xA6DE', 'GB18030-2022'); + +insert into tb_test (id , content) +select 8, convert_from('\xA6DF', 'GB18030-2022'); + +insert into tb_test (id , content) +select 9, convert_from('\xA6EC', 'GB18030-2022'); + +insert into tb_test (id , content) +select 10, convert_from('\xA6ED', 'GB18030-2022'); + +insert into tb_test (id , content) +select 11, convert_from('\xA6F3', 'GB18030-2022'); + +insert into tb_test (id , content) +select 12, convert_from('\xFE59', 'GB18030-2022'); + +insert into tb_test (id , content) +select 13, convert_from('\xFE61', 'GB18030-2022'); + +insert into tb_test (id , content) +select 14, convert_from('\xFE66', 'GB18030-2022'); + +insert into tb_test (id , content) +select 15, convert_from('\xFE67', 'GB18030-2022'); + +insert into tb_test (id , content) +select 16, convert_from('\xFE6D', 'GB18030-2022'); + +insert into tb_test (id , content) +select 17, convert_from('\xFE7E', 'GB18030-2022'); + +insert into tb_test (id , content) +select 18, convert_from('\xFE90', 'GB18030-2022'); + +insert into tb_test (id , content) +select 19, convert_from('\xFEA0', 'GB18030-2022'); + +insert into tb_test (id , content) select 20, convert_from('\x8135F437', 'GB18030-2022'); +insert into tb_test (id , content) select 21, convert_from('\x84318236', 'GB18030-2022'); +insert into tb_test (id , content) select 22, convert_from('\x84318238', 'GB18030-2022'); +insert into tb_test (id , content) select 23, convert_from('\x84318237', 'GB18030-2022'); +insert into tb_test (id , content) select 24, convert_from('\x84318239', 'GB18030-2022'); +insert into tb_test (id , content) select 25, convert_from('\x84318330', 'GB18030-2022'); +insert into tb_test (id , content) select 26, convert_from('\x84318331', 'GB18030-2022'); +insert into tb_test (id , content) select 27, convert_from('\x84318332', 'GB18030-2022'); +insert into tb_test (id , content) select 28, convert_from('\x84318333', 'GB18030-2022'); +insert into tb_test (id , content) select 29, convert_from('\x84318334', 'GB18030-2022'); +insert into tb_test (id , content) select 30, convert_from('\x84318335', 'GB18030-2022'); +insert into tb_test (id , content) select 31, convert_from('\x82359037', 'GB18030-2022'); +insert into tb_test (id , content) select 32, convert_from('\x82359038', 'GB18030-2022'); +insert into tb_test (id , content) select 33, convert_from('\x82359039', 'GB18030-2022'); +insert into tb_test (id , content) select 34, convert_from('\x82359130', 'GB18030-2022'); +insert into tb_test (id , content) select 35, convert_from('\x82359131', 'GB18030-2022'); +insert into tb_test (id , content) select 36, convert_from('\x82359132', 'GB18030-2022'); +insert into tb_test (id , content) select 37, convert_from('\x82359133', 'GB18030-2022'); +insert into tb_test (id , content) select 38, convert_from('\x82359134', 'GB18030-2022'); + +--显示这19个字符 +select * from tb_test order by id; +--查看GB18030-2022编码 +select convert_to(content, 'GB18030-2022') from tb_test order by id; +--查看GB18030-2000编码 +select convert_to(content, 'GB18030') from tb_test order by id; +--转换为UTF-8编码，UTF-8是Unicode的计算机编码形式，想显示为Unicode编码还需进一步转换，数据库中不支持此功能。 +select convert_to(content, 'utf8') from tb_test order by id; + + +--2. 新增字符举例 +--CBJ统一汉字扩充B +select convert_from('\x95328236', 'GB18030-2022'); +select convert_from('\x9835F336', 'GB18030-2022'); +--CJK统一汉字 +select convert_from('\x82358F33', 'GB18030-2022'); +select convert_from('\x82359636', 'GB18030-2022'); +--CJK统一汉子扩充C +select convert_from('\x9835F738', 'GB18030-2022'); +select convert_from('\x98399E36', 'GB18030-2022'); +--CJK统一汉子扩充D +select convert_from('\x98399F38', 'GB18030-2022'); +select convert_from('\x9839B539', 'GB18030-2022'); +--CJK统一汉子扩充E +select convert_from('\x9839B632', 'GB18030-2022'); +select convert_from('\x9933FE33', 'GB18030-2022'); +--CJK统一汉子扩充F +select convert_from('\x99348138', 'GB18030-2022'); +select convert_from('\x9939F730', 'GB18030-2022'); +--康熙部首 +select convert_from('\x81398B32', 'GB18030-2022'); +select convert_from('\x8139A035', 'GB18030-2022'); +--西双版纳新傣文字符 +select convert_from('\x8134F932', 'GB18030-2022'); +select convert_from('\x81358437', 'GB18030-2022'); +--西双版纳老傣文字符 +select convert_from('\x81358B32', 'GB18030-2022'); +select convert_from('\x81359933', 'GB18030-2022'); +--傈僳文字符 +select convert_from('\x82369535', 'GB18030-2022'); +select convert_from('\x82369A32', 'GB18030-2022'); +--蒙古文BIRGA符号 +select convert_from('\x9034C538', 'GB18030-2022'); +select convert_from('\x9034C730', 'GB18030-2022'); +--滇东北苗文字符 +select convert_from('\x9232C636', 'GB18030-2022'); +select convert_from('\x9232D625', 'GB18030-2022'); + +--插入了涉及2000升级到2022新增的字符举例 +create table in_test(id int, content text); + +--CBJ统一汉字扩充B +insert into in_test (id , content) +select 1,convert_from('\x95328236', 'GB18030-2022'); + +insert into in_test (id , content) +select 2,convert_from('\x9835F336', 'GB18030-2022'); + +--CJK统一汉字 +insert into in_test (id , content) +select 3,convert_from('\x82358F33', 'GB18030-2022'); + +insert into in_test (id , content) +select 4,convert_from('\x82359636', 'GB18030-2022'); + +--CJK统一汉子扩充C +insert into in_test (id , content) +select 5,convert_from('\x9835F738', 'GB18030-2022'); + +insert into in_test (id , content) +select 6,convert_from('\x98399E36', 'GB18030-2022'); + +--CJK统一汉子扩充D +insert into in_test (id , content) +select 7,convert_from('\x98399F38', 'GB18030-2022'); + +insert into in_test (id , content) +select 8,convert_from('\x9839B539', 'GB18030-2022'); + +--CJK统一汉子扩充E +insert into in_test (id , content) +select 9,convert_from('\x9839B632', 'GB18030-2022'); + +insert into in_test (id , content) +select 10,convert_from('\x9933FE33', 'GB18030-2022'); + +--CJK统一汉子扩充F +insert into in_test (id , content) +select 11,convert_from('\x99348138', 'GB18030-2022'); + +insert into in_test (id , content) +select 12,convert_from('\x9939F730', 'GB18030-2022'); + +--康熙部首 +insert into in_test (id , content) +select 13,convert_from('\x81398B32', 'GB18030-2022'); + +insert into in_test (id , content) +select 14,convert_from('\x8139A035', 'GB18030-2022'); + +--西双版纳新傣文字符 +insert into in_test (id , content) +select 15,convert_from('\x8134F932', 'GB18030-2022'); + +insert into in_test (id , content) +select 16,convert_from('\x81358437', 'GB18030-2022'); + +--西双版纳老傣文字符 +insert into in_test (id , content) +select 17,convert_from('\x81358B32', 'GB18030-2022'); + +insert into in_test (id , content) +select 18,convert_from('\x81359933', 'GB18030-2022'); + +--傈僳文字符 +insert into in_test (id , content) +select 19,convert_from('\x82369535', 'GB18030-2022'); + +insert into in_test (id , content) +select 20,convert_from('\x82369A32', 'GB18030-2022'); + +--蒙古文BIRGA符号 +insert into in_test (id , content) +select 21,convert_from('\x9034C538', 'GB18030-2022'); + +insert into in_test (id , content) +select 22,convert_from('\x9034C730', 'GB18030-2022'); + +--滇东北苗文字符 +insert into in_test (id , content) +select 23,convert_from('\x9232C636', 'GB18030-2022'); + +insert into in_test (id , content) +select 24,convert_from('\x9232D625', 'GB18030-2022'); + +--显示这24个字符 +select * from in_test order by id; +--查看GB18030-2022编码 +select convert_to(content, 'GB18030-2022') from in_test order by id; +--转换为UTF-8编码，UTF-8是Unicode的计算机编码形式，想显示为Unicode编码还需进一步转换，数据库中不支持此功能。 +select convert_to(content, 'utf8') from in_test order by id; + +drop table in_test; +drop table tb_test; + +--3. 正常的增删改查语句测试 +create table 表1(id int, 人名 name); + +insert into 表1(id, 人名) values(1, '小明！'); +select * from 表1; + +alter table 表1 drop 人名; +select * from 表1; + +alter table 表1 add 学校 text; +insert into 表1(id , 学校) select 2, convert_to('@华为大学￥', 'GB18030-2022'); +select * from 表1; + +drop table 表1; + +--4. 不存在映射关系时 +select convert('\xFD308130', 'GB18030-2022', 'UTF8'); +select convert('\xFE39FE39', 'GB18030-2022', 'UTF8'); + +--5. 测试gb18030_2022数据库中的字符串相关 +-- E021-03 character string literals +SELECT 'first line' +' - next line' + ' - third line' + AS "Three lines to one"; + +-- illegal string continuation syntax +SELECT 'first line' +' - next line' /* this comment is not allowed here */ +' - third line' + AS "Illegal comment within continuation"; + +-- Unicode escapes +SET standard_conforming_strings TO on; + +SELECT U&'d\0061t\+000061' AS U&"d\0061t\+000061"; +SELECT U&'d!0061t\+000061' UESCAPE '!' AS U&"d*0061t\+000061" UESCAPE '*'; + +-- bytea +SET bytea_output TO hex; +SELECT E'\\xDeAdBeEf'::bytea; +SELECT E'\\x De Ad Be Ef '::bytea; +SELECT E'\\xDeAdBeE'::bytea; +SELECT E'\\xDeAdBeEx'::bytea; +SELECT E'\\xDe00BeEf'::bytea; +SELECT E'DeAdBeEf'::bytea; +SELECT E'De\\000dBeEf'::bytea; +SELECT E'De\123dBeEf'::bytea; +SELECT E'De\\123dBeEf'::bytea; +SELECT E'De\\678dBeEf'::bytea; + +SET bytea_output TO escape; +SELECT E'\\xDeAdBeEf'::bytea; +SELECT E'\\x De Ad Be Ef '::bytea; +SELECT E'\\xDe00BeEf'::bytea; +SELECT E'DeAdBeEf'::bytea; +SELECT E'De\\000dBeEf'::bytea; +SELECT E'De\\123dBeEf'::bytea; + +SET bytea_output TO hex; + +SELECT CAST(name 'namefield' AS text) AS "text(name)"; + +-- E021-09 trim function +SELECT TRIM(BOTH FROM ' bunch o blanks ') = 'bunch o blanks' AS "bunch o blanks"; + +-- E021-06 substring expression +SELECT SUBSTRING('1234567890' FROM 3) = '34567890' AS "34567890"; + +-- PostgreSQL extension to allow using back reference in replace string; +SELECT regexp_replace('1112223333', E'(\\d{3})(\\d{3})(\\d{4})', E'(\\1) \\2-\\3'); + +-- set so we can tell NULL from empty string +\pset null '\\N' + +-- return all matches from regexp +SELECT regexp_matches('foobarbequebaz', $re$(bar)(beque)$re$); + +-- split string on regexp +SELECT foo, length(foo) FROM regexp_split_to_table('the quick brown fox jumped over the lazy dog', $re$\s+$re$) AS foo; +SELECT regexp_split_to_array('the quick brown fox jumped over the lazy dog', $re$\s+$re$); + +-- change NULL-display back +\pset null '' + +-- E021-11 position expression +SELECT POSITION('4' IN '1234567890') = '4' AS "4"; + +SELECT POSITION('5' IN '1234567890') = '5' AS "5"; + +-- T312 character overlay function +SELECT OVERLAY('abcdef' PLACING '45' FROM 4) AS "abc45f"; + +-- E061-04 like predicate +SELECT 'hawkeye' LIKE 'h%' AS "true"; +SELECT 'hawkeye' NOT LIKE 'h%' AS "false"; + +-- unused escape character +SELECT 'hawkeye' LIKE 'h%' ESCAPE '#' AS "true"; +SELECT 'hawkeye' NOT LIKE 'h%' ESCAPE '#' AS "false"; + +-- +-- test ILIKE (case-insensitive LIKE) +-- Be sure to form every test as an ILIKE/NOT ILIKE pair. +-- + +SELECT 'hawkeye' ILIKE 'h%' AS "true"; +SELECT 'hawkeye' NOT ILIKE 'h%' AS "false"; + +--6. 使用字符串相关函数 +--重复字符串 +select repeat('中国', 3); + +--返回字符串的前n个字符 +select left('中国！number1', 7); + +--返回长度 +select length('中国！number1'); + +--反转字符串 +select reverse('中国！number1'); + +--md5算法加密 +select md5('中国！number1'); + +-- test strpos +SELECT strpos('abcdef', 'cd') AS "pos_3"; +SELECT strpos('abcdef', 'xy') AS "pos_0"; + +SELECT replace('yabadabadoo', 'ba', '123') AS "ya123da123doo"; + +select split_part('joeuser@mydatabase','@',3) AS "empty string"; + +select to_hex(256::bigint*256::bigint*256::bigint*256::bigint - 1) AS "ffffffff"; + +--返回字符串中第一个字符的十进制表示形式 +select ascii('xyz'); +select ascii('中xyz'); +select ascii('ḿxyz'); + +-- 7. 检查GB18030-2022与GB18030的关系 +select convert('中国', 'GB18030', 'GB18030-2022'); + +select convert('中国', 'GB18030-2022', 'GB18030'); + +select convert('\xA8BC', 'GB18030-2022', 'UTF8'); + +select convert('\xA8BC', 'GB18030', 'UTF8'); + +\c regression +clean connection to all force for database gb18030_2022; +drop database gb18030_2022; + +-- 8. 在UTF8环境下检测GB18030-2022与UTF8的转换 +select convert('中国&华为*GaussDB', 'UTF8', 'GB18030-2022'); + +select convert('ḿ', 'UTF8', 'GB18030-2022'); + +-- 9. 测试create database时encoding与本地设置不匹配 +create database gb18030_2022 encoding='gb18030-2022' LC_COLLATE='en_US.utf-8' LC_CTYPE ='en_US.utf-8' TEMPLATE=template0; + +--10. 测试initdb +\! rm -f @abs_bindir@/test_initdb.log +\! mkdir -p @testtablespace@/test2 +\! mkdir -p @testtablespace@/test2/pg_location +\! @abs_bindir@/gs_initdb -S -D @testtablespace@/test2 --nodename coorn2 -U test_initdb -w test@123 --locale=zh_CN.gb18030 -E GB18030_2022 >> @abs_bindir@/test_initdb2.log 2>&1 +\! cat @abs_bindir@/test_initdb2.log | grep ok +\! rm -f @abs_bindir@/test_initdb2.log +\! rm -rf @testtablespace@/test2 + +--11. 测试升级回滚 +select oid, * from pg_conversion where conname like '%gb18030%' order by conname; + +select oid, * from pg_proc where proname like '%gb18030%' order by proname; + +--12. 看护client_encoding不能设置为GB18030_2022 +set client_encoding = GB18030_2022; + +ALTER SESSION SET NAMES 'GB18030_2022'; \ No newline at end of file diff --git a/src/test/regress/output/ts_gb18030_utf8.source b/src/test/regress/output/ts_gb18030_utf8.source new file mode 100644 index 000000000..d712a77da --- /dev/null +++ b/src/test/regress/output/ts_gb18030_utf8.source @@ -0,0 +1,1142 @@ +create database gb18030_2022 encoding='gb18030-2022' LC_COLLATE='zh_CN.GB18030' LC_CTYPE ='zh_CN.GB18030' TEMPLATE=template0; +\c gb18030_2022 + +show server_encoding; + server_encoding +----------------- + GB18030_2022 +(1 row) + +set client_encoding = 'UTF8'; +show client_encoding; + client_encoding +----------------- + UTF8 +(1 row) + +--1. 编码映射关系发生改变的字符： +-- 查看未发生变化的字符串： +select convert_to('中国', 'GB18030-2022'); + convert_to +------------ + \xd6d0b9fa +(1 row) + + +--插入了涉及GB18030-2000升级GB18030-2022后GB18030与UTF-8转换关系变更的19个字符。 +create table tb_test(id int, content text); + +insert into tb_test (id , content) +select 1, convert_from('\xA8BC', 'GB18030-2022'); + +insert into tb_test (id , content) +select 2, convert_from('\xA6D9', 'GB18030-2022'); + +insert into tb_test (id , content) +select 3, convert_from('\xA6DA', 'GB18030-2022'); + +insert into tb_test (id , content) +select 4, convert_from('\xA6DB', 'GB18030-2022'); + +insert into tb_test (id , content) +select 5, convert_from('\xA6DC', 'GB18030-2022'); + +insert into tb_test (id , content) +select 6, convert_from('\xA6DD', 'GB18030-2022'); + +insert into tb_test (id , content) +select 7, convert_from('\xA6DE', 'GB18030-2022'); + +insert into tb_test (id , content) +select 8, convert_from('\xA6DF', 'GB18030-2022'); + +insert into tb_test (id , content) +select 9, convert_from('\xA6EC', 'GB18030-2022'); + +insert into tb_test (id , content) +select 10, convert_from('\xA6ED', 'GB18030-2022'); + +insert into tb_test (id , content) +select 11, convert_from('\xA6F3', 'GB18030-2022'); + +insert into tb_test (id , content) +select 12, convert_from('\xFE59', 'GB18030-2022'); + +insert into tb_test (id , content) +select 13, convert_from('\xFE61', 'GB18030-2022'); + +insert into tb_test (id , content) +select 14, convert_from('\xFE66', 'GB18030-2022'); + +insert into tb_test (id , content) +select 15, convert_from('\xFE67', 'GB18030-2022'); + +insert into tb_test (id , content) +select 16, convert_from('\xFE6D', 'GB18030-2022'); + +insert into tb_test (id , content) +select 17, convert_from('\xFE7E', 'GB18030-2022'); + +insert into tb_test (id , content) +select 18, convert_from('\xFE90', 'GB18030-2022'); + +insert into tb_test (id , content) +select 19, convert_from('\xFEA0', 'GB18030-2022'); + +insert into tb_test (id , content) select 20, convert_from('\x8135F437', 'GB18030-2022'); +insert into tb_test (id , content) select 21, convert_from('\x84318236', 'GB18030-2022'); +insert into tb_test (id , content) select 22, convert_from('\x84318238', 'GB18030-2022'); +insert into tb_test (id , content) select 23, convert_from('\x84318237', 'GB18030-2022'); +insert into tb_test (id , content) select 24, convert_from('\x84318239', 'GB18030-2022'); +insert into tb_test (id , content) select 25, convert_from('\x84318330', 'GB18030-2022'); +insert into tb_test (id , content) select 26, convert_from('\x84318331', 'GB18030-2022'); +insert into tb_test (id , content) select 27, convert_from('\x84318332', 'GB18030-2022'); +insert into tb_test (id , content) select 28, convert_from('\x84318333', 'GB18030-2022'); +insert into tb_test (id , content) select 29, convert_from('\x84318334', 'GB18030-2022'); +insert into tb_test (id , content) select 30, convert_from('\x84318335', 'GB18030-2022'); +insert into tb_test (id , content) select 31, convert_from('\x82359037', 'GB18030-2022'); +insert into tb_test (id , content) select 32, convert_from('\x82359038', 'GB18030-2022'); +insert into tb_test (id , content) select 33, convert_from('\x82359039', 'GB18030-2022'); +insert into tb_test (id , content) select 34, convert_from('\x82359130', 'GB18030-2022'); +insert into tb_test (id , content) select 35, convert_from('\x82359131', 'GB18030-2022'); +insert into tb_test (id , content) select 36, convert_from('\x82359132', 'GB18030-2022'); +insert into tb_test (id , content) select 37, convert_from('\x82359133', 'GB18030-2022'); +insert into tb_test (id , content) select 38, convert_from('\x82359134', 'GB18030-2022'); + +--显示这19个字符 +select * from tb_test order by id; + id | content +----+--------- + 1 | ḿ + 2 | ︐ + 3 | ︒ + 4 | ︑ + 5 | ︓ + 6 | ︔ + 7 | ︕ + 8 | ︖ + 9 | ︗ + 10 | ︘ + 11 | ︙ + 12 | 龴 + 13 | 龵 + 14 | 龶 + 15 | 龷 + 16 | 龸 + 17 | 龹 + 18 | 龺 + 19 | 龻 + 20 |  + 21 |  + 22 |  + 23 |  + 24 |  + 25 |  + 26 |  + 27 |  + 28 |  + 29 |  + 30 |  + 31 |  + 32 |  + 33 |  + 34 |  + 35 |  + 36 |  + 37 |  + 38 |  +(38 rows) + +--查看GB18030-2022编码 +select convert_to(content, 'GB18030-2022') from tb_test order by id; + convert_to +------------ + \xa8bc + \xa6d9 + \xa6da + \xa6db + \xa6dc + \xa6dd + \xa6de + \xa6df + \xa6ec + \xa6ed + \xa6f3 + \xfe59 + \xfe61 + \xfe66 + \xfe67 + \xfe6d + \xfe7e + \xfe90 + \xfea0 + \x8135f437 + \x84318236 + \x84318238 + \x84318237 + \x84318239 + \x84318330 + \x84318331 + \x84318332 + \x84318333 + \x84318334 + \x84318335 + \x82359037 + \x82359038 + \x82359039 + \x82359130 + \x82359131 + \x82359132 + \x82359133 + \x82359134 +(38 rows) + +--查看GB18030-2000编码 +select convert_to(content, 'GB18030') from tb_test order by id; + convert_to +------------ + \xa8bc + \xa6d9 + \xa6da + \xa6db + \xa6dc + \xa6dd + \xa6de + \xa6df + \xa6ec + \xa6ed + \xa6f3 + \xfe59 + \xfe61 + \xfe66 + \xfe67 + \xfe6d + \xfe7e + \xfe90 + \xfea0 + \x8135f437 + \x84318236 + \x84318238 + \x84318237 + \x84318239 + \x84318330 + \x84318331 + \x84318332 + \x84318333 + \x84318334 + \x84318335 + \x82359037 + \x82359038 + \x82359039 + \x82359130 + \x82359131 + \x82359132 + \x82359133 + \x82359134 +(38 rows) + +--转换为UTF-8编码，UTF-8是Unicode的计算机编码形式，想显示为Unicode编码还需进一步转换，数据库中不支持此功能。 +select convert_to(content, 'utf8') from tb_test order by id; + convert_to +------------ + \xe1b8bf + \xefb890 + \xefb892 + \xefb891 + \xefb893 + \xefb894 + \xefb895 + \xefb896 + \xefb897 + \xefb898 + \xefb899 + \xe9beb4 + \xe9beb5 + \xe9beb6 + \xe9beb7 + \xe9beb8 + \xe9beb9 + \xe9beba + \xe9bebb + \xee9f87 + \xee9e8d + \xee9e8e + \xee9e8f + \xee9e90 + \xee9e91 + \xee9e92 + \xee9e93 + \xee9e94 + \xee9e95 + \xee9e96 + \xeea09e + \xeea0a6 + \xeea0ab + \xeea0ac + \xeea0b2 + \xeea183 + \xeea194 + \xeea1a4 +(38 rows) + + + +--2. 新增字符举例 +--CBJ统一汉字扩充B +select convert_from('\x95328236', 'GB18030-2022'); + convert_from +-------------- + 𠀀 +(1 row) + +select convert_from('\x9835F336', 'GB18030-2022'); + convert_from +-------------- + 𪛖 +(1 row) + +--CJK统一汉字 +select convert_from('\x82358F33', 'GB18030-2022'); + convert_from +-------------- + 龦 +(1 row) + +select convert_from('\x82359636', 'GB18030-2022'); + convert_from +-------------- + 鿯 +(1 row) + +--CJK统一汉子扩充C +select convert_from('\x9835F738', 'GB18030-2022'); + convert_from +-------------- + 𪜀 +(1 row) + +select convert_from('\x98399E36', 'GB18030-2022'); + convert_from +-------------- + 𫜴 +(1 row) + +--CJK统一汉子扩充D +select convert_from('\x98399F38', 'GB18030-2022'); + convert_from +-------------- + 𫝀 +(1 row) + +select convert_from('\x9839B539', 'GB18030-2022'); + convert_from +-------------- + 𫠝 +(1 row) + +--CJK统一汉子扩充E +select convert_from('\x9839B632', 'GB18030-2022'); + convert_from +-------------- + 𫠠 +(1 row) + +select convert_from('\x9933FE33', 'GB18030-2022'); + convert_from +-------------- + 𬺡 +(1 row) + +--CJK统一汉子扩充F +select convert_from('\x99348138', 'GB18030-2022'); + convert_from +-------------- + 𬺰 +(1 row) + +select convert_from('\x9939F730', 'GB18030-2022'); + convert_from +-------------- + 𮯠 +(1 row) + +--康熙部首 +select convert_from('\x81398B32', 'GB18030-2022'); + convert_from +-------------- + ⼀ +(1 row) + +select convert_from('\x8139A035', 'GB18030-2022'); + convert_from +-------------- + ⿕ +(1 row) + +--西双版纳新傣文字符 +select convert_from('\x8134F932', 'GB18030-2022'); + convert_from +-------------- + ᦀ +(1 row) + +select convert_from('\x81358437', 'GB18030-2022'); + convert_from +-------------- + ᧟ +(1 row) + +--西双版纳老傣文字符 +select convert_from('\x81358B32', 'GB18030-2022'); + convert_from +-------------- + ᨠ +(1 row) + +select convert_from('\x81359933', 'GB18030-2022'); + convert_from +-------------- + ᪭ +(1 row) + +--傈僳文字符 +select convert_from('\x82369535', 'GB18030-2022'); + convert_from +-------------- + ꓐ +(1 row) + +select convert_from('\x82369A32', 'GB18030-2022'); + convert_from +-------------- + ꓿ +(1 row) + +--蒙古文BIRGA符号 +select convert_from('\x9034C538', 'GB18030-2022'); + convert_from +-------------- + 𑙠 +(1 row) + +select convert_from('\x9034C730', 'GB18030-2022'); + convert_from +-------------- + 𑙬 +(1 row) + +--滇东北苗文字符 +select convert_from('\x9232C636', 'GB18030-2022'); + convert_from +-------------- + 𖼀 +(1 row) + +select convert_from('\x9232D625', 'GB18030-2022'); + convert_from +-------------- + 𖾏 +(1 row) + + +--插入了涉及2000升级到2022新增的字符举例 +create table in_test(id int, content text); + +--CBJ统一汉字扩充B +insert into in_test (id , content) +select 1,convert_from('\x95328236', 'GB18030-2022'); + +insert into in_test (id , content) +select 2,convert_from('\x9835F336', 'GB18030-2022'); + +--CJK统一汉字 +insert into in_test (id , content) +select 3,convert_from('\x82358F33', 'GB18030-2022'); + +insert into in_test (id , content) +select 4,convert_from('\x82359636', 'GB18030-2022'); + +--CJK统一汉子扩充C +insert into in_test (id , content) +select 5,convert_from('\x9835F738', 'GB18030-2022'); + +insert into in_test (id , content) +select 6,convert_from('\x98399E36', 'GB18030-2022'); + +--CJK统一汉子扩充D +insert into in_test (id , content) +select 7,convert_from('\x98399F38', 'GB18030-2022'); + +insert into in_test (id , content) +select 8,convert_from('\x9839B539', 'GB18030-2022'); + +--CJK统一汉子扩充E +insert into in_test (id , content) +select 9,convert_from('\x9839B632', 'GB18030-2022'); + +insert into in_test (id , content) +select 10,convert_from('\x9933FE33', 'GB18030-2022'); + +--CJK统一汉子扩充F +insert into in_test (id , content) +select 11,convert_from('\x99348138', 'GB18030-2022'); + +insert into in_test (id , content) +select 12,convert_from('\x9939F730', 'GB18030-2022'); + +--康熙部首 +insert into in_test (id , content) +select 13,convert_from('\x81398B32', 'GB18030-2022'); + +insert into in_test (id , content) +select 14,convert_from('\x8139A035', 'GB18030-2022'); + +--西双版纳新傣文字符 +insert into in_test (id , content) +select 15,convert_from('\x8134F932', 'GB18030-2022'); + +insert into in_test (id , content) +select 16,convert_from('\x81358437', 'GB18030-2022'); + +--西双版纳老傣文字符 +insert into in_test (id , content) +select 17,convert_from('\x81358B32', 'GB18030-2022'); + +insert into in_test (id , content) +select 18,convert_from('\x81359933', 'GB18030-2022'); + +--傈僳文字符 +insert into in_test (id , content) +select 19,convert_from('\x82369535', 'GB18030-2022'); + +insert into in_test (id , content) +select 20,convert_from('\x82369A32', 'GB18030-2022'); + +--蒙古文BIRGA符号 +insert into in_test (id , content) +select 21,convert_from('\x9034C538', 'GB18030-2022'); + +insert into in_test (id , content) +select 22,convert_from('\x9034C730', 'GB18030-2022'); + +--滇东北苗文字符 +insert into in_test (id , content) +select 23,convert_from('\x9232C636', 'GB18030-2022'); + +insert into in_test (id , content) +select 24,convert_from('\x9232D625', 'GB18030-2022'); + +--显示这24个字符 +select * from in_test order by id; + id | content +----+--------- + 1 | 𠀀 + 2 | 𪛖 + 3 | 龦 + 4 | 鿯 + 5 | 𪜀 + 6 | 𫜴 + 7 | 𫝀 + 8 | 𫠝 + 9 | 𫠠 + 10 | 𬺡 + 11 | 𬺰 + 12 | 𮯠 + 13 | ⼀ + 14 | ⿕ + 15 | ᦀ + 16 | ᧟ + 17 | ᨠ + 18 | ᪭ + 19 | ꓐ + 20 | ꓿ + 21 | 𑙠 + 22 | 𑙬 + 23 | 𖼀 + 24 | 𖾏 +(24 rows) + +--查看GB18030-2022编码 +select convert_to(content, 'GB18030-2022') from in_test order by id; + convert_to +------------ + \x95328236 + \x9835f336 + \x82358f33 + \x82359636 + \x9835f738 + \x98399e36 + \x98399f38 + \x9839b539 + \x9839b632 + \x9933fe33 + \x99348138 + \x9939f730 + \x81398b32 + \x8139a035 + \x8134f932 + \x81358437 + \x81358b32 + \x81359933 + \x82369535 + \x82369a32 + \x9034c538 + \x9034c730 + \x9232c636 + \x9232d625 +(24 rows) + +--转换为UTF-8编码，UTF-8是Unicode的计算机编码形式，想显示为Unicode编码还需进一步转换，数据库中不支持此功能。 +select convert_to(content, 'utf8') from in_test order by id; + convert_to +------------ + \xf0a08080 + \xf0aa9b96 + \xe9bea6 + \xe9bfaf + \xf0aa9c80 + \xf0ab9cb4 + \xf0ab9d80 + \xf0aba09d + \xf0aba0a0 + \xf0acbaa1 + \xf0acbab0 + \xf0aeafa0 + \xe2bc80 + \xe2bf95 + \xe1a680 + \xe1a79f + \xe1a8a0 + \xe1aaad + \xea9390 + \xea93bf + \xf09199a0 + \xf09199ac + \xf096bc80 + \xf096be8f +(24 rows) + + +drop table in_test; +drop table tb_test; + +--3. 正常的增删改查语句测试 +create table 表1(id int, 人名 name); + +insert into 表1(id, 人名) values(1, '小明！'); +select * from 表1; + id | 人名 +----+-------- + 1 | 小明！ +(1 row) + + +alter table 表1 drop 人名; +select * from 表1; + id +---- + 1 +(1 row) + + +alter table 表1 add 学校 text; +insert into 表1(id , 学校) select 2, convert_to('@华为大学￥', 'GB18030-2022'); +select * from 表1; + id | 学校 +----+-------------------------- + 1 | + 2 | \x40bbaaceaab4f3d1a7a3a4 +(2 rows) + + +drop table 表1; + +--4. 不存在映射关系时 +select convert('\xFD308130', 'GB18030-2022', 'UTF8'); +ERROR: character with byte sequence 0xfd 0x30 0x81 0x30 in encoding "GB18030_2022" has no equivalent in encoding "UTF8" +CONTEXT: referenced column: convert +select convert('\xFE39FE39', 'GB18030-2022', 'UTF8'); +ERROR: character with byte sequence 0xfe 0x39 0xfe 0x39 in encoding "GB18030_2022" has no equivalent in encoding "UTF8" +CONTEXT: referenced column: convert + +--5. 测试gb18030_2022数据库中的字符串相关 +-- E021-03 character string literals +SELECT 'first line' +' - next line' + ' - third line' + AS "Three lines to one"; + Three lines to one +------------------------------------- + first line - next line - third line +(1 row) + + +-- illegal string continuation syntax +SELECT 'first line' +' - next line' /* this comment is not allowed here */ +' - third line' + AS "Illegal comment within continuation"; +ERROR: syntax error at or near "' - third line'" +LINE 3: ' - third line' + ^ + +-- Unicode escapes +SET standard_conforming_strings TO on; + +SELECT U&'d\0061t\+000061' AS U&"d\0061t\+000061"; + data +------ + data +(1 row) + +SELECT U&'d!0061t\+000061' UESCAPE '!' AS U&"d*0061t\+000061" UESCAPE '*'; + dat\+000061 +------------- + dat\+000061 +(1 row) + + +-- bytea +SET bytea_output TO hex; +SELECT E'\\xDeAdBeEf'::bytea; + bytea +------------ + \xdeadbeef +(1 row) + +SELECT E'\\x De Ad Be Ef '::bytea; + bytea +------------ + \xdeadbeef +(1 row) + +SELECT E'\\xDeAdBeE'::bytea; +ERROR: invalid hexadecimal data: odd number of digits +LINE 1: SELECT E'\\xDeAdBeE'::bytea; + ^ +CONTEXT: referenced column: bytea +SELECT E'\\xDeAdBeEx'::bytea; +ERROR: invalid hexadecimal digit: "x" +LINE 1: SELECT E'\\xDeAdBeEx'::bytea; + ^ +CONTEXT: referenced column: bytea +SELECT E'\\xDe00BeEf'::bytea; + bytea +------------ + \xde00beef +(1 row) + +SELECT E'DeAdBeEf'::bytea; + bytea +-------------------- + \x4465416442654566 +(1 row) + +SELECT E'De\\000dBeEf'::bytea; + bytea +-------------------- + \x4465006442654566 +(1 row) + +SELECT E'De\123dBeEf'::bytea; + bytea +-------------------- + \x4465536442654566 +(1 row) + +SELECT E'De\\123dBeEf'::bytea; + bytea +-------------------- + \x4465536442654566 +(1 row) + +SELECT E'De\\678dBeEf'::bytea; +ERROR: invalid input syntax for type bytea +LINE 1: SELECT E'De\\678dBeEf'::bytea; + ^ +CONTEXT: referenced column: bytea + +SET bytea_output TO escape; +SELECT E'\\xDeAdBeEf'::bytea; + bytea +------------------ + \336\255\276\357 +(1 row) + +SELECT E'\\x De Ad Be Ef '::bytea; + bytea +------------------ + \336\255\276\357 +(1 row) + +SELECT E'\\xDe00BeEf'::bytea; + bytea +------------------ + \336\000\276\357 +(1 row) + +SELECT E'DeAdBeEf'::bytea; + bytea +---------- + DeAdBeEf +(1 row) + +SELECT E'De\\000dBeEf'::bytea; + bytea +------------- + De\000dBeEf +(1 row) + +SELECT E'De\\123dBeEf'::bytea; + bytea +---------- + DeSdBeEf +(1 row) + + +SET bytea_output TO hex; + +SELECT CAST(name 'namefield' AS text) AS "text(name)"; + text(name) +------------ + namefield +(1 row) + + +-- E021-09 trim function +SELECT TRIM(BOTH FROM ' bunch o blanks ') = 'bunch o blanks' AS "bunch o blanks"; + bunch o blanks +---------------- + t +(1 row) + + +-- E021-06 substring expression +SELECT SUBSTRING('1234567890' FROM 3) = '34567890' AS "34567890"; + 34567890 +---------- + t +(1 row) + + +-- PostgreSQL extension to allow using back reference in replace string; +SELECT regexp_replace('1112223333', E'(\\d{3})(\\d{3})(\\d{4})', E'(\\1) \\2-\\3'); + regexp_replace +---------------- + (111) 222-3333 +(1 row) + + +-- set so we can tell NULL from empty string +\pset null '\\N' + +-- return all matches from regexp +SELECT regexp_matches('foobarbequebaz', $re$(bar)(beque)$re$); + regexp_matches +---------------- + {bar,beque} +(1 row) + + +-- split string on regexp +SELECT foo, length(foo) FROM regexp_split_to_table('the quick brown fox jumped over the lazy dog', $re$\s+$re$) AS foo; + foo | length +--------+-------- + the | 3 + quick | 5 + brown | 5 + fox | 3 + jumped | 6 + over | 4 + the | 3 + lazy | 4 + dog | 3 +(9 rows) + +SELECT regexp_split_to_array('the quick brown fox jumped over the lazy dog', $re$\s+$re$); + regexp_split_to_array +------------------------------------------------ + {the,quick,brown,fox,jumped,over,the,lazy,dog} +(1 row) + + +-- change NULL-display back +\pset null '' + +-- E021-11 position expression +SELECT POSITION('4' IN '1234567890') = '4' AS "4"; + 4 +--- + t +(1 row) + + +SELECT POSITION('5' IN '1234567890') = '5' AS "5"; + 5 +--- + t +(1 row) + + +-- T312 character overlay function +SELECT OVERLAY('abcdef' PLACING '45' FROM 4) AS "abc45f"; + abc45f +-------- + abc45f +(1 row) + + +-- E061-04 like predicate +SELECT 'hawkeye' LIKE 'h%' AS "true"; + true +------ + t +(1 row) + +SELECT 'hawkeye' NOT LIKE 'h%' AS "false"; + false +------- + f +(1 row) + + +-- unused escape character +SELECT 'hawkeye' LIKE 'h%' ESCAPE '#' AS "true"; + true +------ + t +(1 row) + +SELECT 'hawkeye' NOT LIKE 'h%' ESCAPE '#' AS "false"; + false +------- + f +(1 row) + + +-- +-- test ILIKE (case-insensitive LIKE) +-- Be sure to form every test as an ILIKE/NOT ILIKE pair. +-- + +SELECT 'hawkeye' ILIKE 'h%' AS "true"; + true +------ + t +(1 row) + +SELECT 'hawkeye' NOT ILIKE 'h%' AS "false"; + false +------- + f +(1 row) + + +--6. 使用字符串相关函数 +--重复字符串 +select repeat('中国', 3); + repeat +-------------- + 中国中国中国 +(1 row) + + +--返回字符串的前n个字符 +select left('中国！number1', 7); + left +------------ + 中国！numb +(1 row) + + +--返回长度 +select length('中国！number1'); + length +-------- + 10 +(1 row) + + +--反转字符串 +select reverse('中国！number1'); + reverse +--------------- + 1rebmun！国中 +(1 row) + + +--md5算法加密 +select md5('中国！number1'); + md5 +---------------------------------- + 764c69059680eb8f52946f9f4936737a +(1 row) + + +-- test strpos +SELECT strpos('abcdef', 'cd') AS "pos_3"; + pos_3 +------- + 3 +(1 row) + +SELECT strpos('abcdef', 'xy') AS "pos_0"; + pos_0 +------- + 0 +(1 row) + + +SELECT replace('yabadabadoo', 'ba', '123') AS "ya123da123doo"; + ya123da123doo +--------------- + ya123da123doo +(1 row) + + +select split_part('joeuser@mydatabase','@',3) AS "empty string"; + empty string +-------------- + +(1 row) + + +select to_hex(256::bigint*256::bigint*256::bigint*256::bigint - 1) AS "ffffffff"; + ffffffff +---------- + ffffffff +(1 row) + + +--返回字符串中第一个字符的十进制表示形式 +select ascii('xyz'); + ascii +------- + 120 +(1 row) + +select ascii('中xyz'); +ERROR: requested character too large +CONTEXT: referenced column: ascii +select ascii('ḿxyz'); +ERROR: requested character too large +CONTEXT: referenced column: ascii + +-- 7. 检查GB18030-2022与GB18030的关系 +select convert('中国', 'GB18030', 'GB18030-2022'); + convert +------------ + \xd6d0b9fa +(1 row) + + +select convert('中国', 'GB18030-2022', 'GB18030'); + convert +------------ + \xd6d0b9fa +(1 row) + + +select convert('\xA8BC', 'GB18030-2022', 'UTF8'); + convert +---------- + \xe1b8bf +(1 row) + + +select convert('\xA8BC', 'GB18030', 'UTF8'); + convert +---------- + \xee9f87 +(1 row) + + +\c regression +clean connection to all force for database gb18030_2022; +drop database gb18030_2022; + +-- 8. 在UTF8环境下检测GB18030-2022与UTF8的转换 +select convert('中国&华为*GaussDB', 'UTF8', 'GB18030-2022'); + convert +-------------------------------------- + \xd6d0b9fa26bbaaceaa2a47617573734442 +(1 row) + + +select convert('ḿ', 'UTF8', 'GB18030-2022'); + convert +--------- + \xa8bc +(1 row) + + +-- 9. 测试create database时encoding与本地设置不匹配 +create database gb18030_2022 encoding='gb18030-2022' LC_COLLATE='en_US.utf-8' LC_CTYPE ='en_US.utf-8' TEMPLATE=template0; +ERROR: encoding "GB18030_2022" does not match locale "en_US.utf-8" +DETAIL: The chosen LC_CTYPE setting requires encoding "UTF8". + +--10. 测试initdb +\! rm -f @abs_bindir@/test_initdb.log +\! mkdir -p @testtablespace@/test2 +\! mkdir -p @testtablespace@/test2/pg_location +\! @abs_bindir@/gs_initdb -S -D @testtablespace@/test2 --nodename coorn2 -U test_initdb -w test@123 --locale=zh_CN.gb18030 -E GB18030_2022 >> @abs_bindir@/test_initdb2.log 2>&1 +\! cat @abs_bindir@/test_initdb2.log | grep ok +ok +creating subdirectories ... in ordinary occasionok +creating configuration files ... ok +ok +initializing pg_authid ... ok +setting password ... ok +initializing dependencies ... ok +loading PL/pgSQL server-side language ... ok +creating system views ... ok +creating performance views ... ok +loading system objects' descriptions ... ok +creating collations ... ok +creating conversions ... ok +creating dictionaries ... ok +setting privileges on built-in objects ... ok +initialize global configure for bucketmap length ... ok +creating information schema ... ok +loading foreign-data wrapper for distfs access ... ok +loading foreign-data wrapper for log access ... ok +loading hstore extension ... ok +loading security plugin ... ok +update system tables ... ok +creating snapshots catalog ... ok +vacuuming database template1 ... ok +copying template1 to template0 ... ok +copying template1 to postgres ... ok +freezing database template0 ... ok +freezing database template1 ... ok +freezing database postgres ... ok +\! rm -f @abs_bindir@/test_initdb2.log +\! rm -rf @testtablespace@/test2 + +--11. 测试升级回滚 +select oid, * from pg_conversion where conname like '%gb18030%' order by conname; + oid | conname | connamespace | conowner | conforencoding | contoencoding | conproc | condefault +-------+----------------------+--------------+----------+----------------+---------------+----------------------+------------ +--? .* | gb18030_2022_to_utf8 | 11 | 10 | 37 | 7 | gb18030_2022_to_utf8 | t +--? .* | gb18030_to_utf8 | 11 | 10 | 36 | 7 | gb18030_to_utf8 | t +--? .* | utf8_to_gb18030 | 11 | 10 | 7 | 36 | utf8_to_gb18030 | t +--? .* | utf8_to_gb18030_2022 | 11 | 10 | 7 | 37 | utf8_to_gb18030_2022 | t +(4 rows) + + +select oid, * from pg_proc where proname like '%gb18030%' order by proname; + oid | proname | pronamespace | proowner | prolang | procost | prorows | provariadic | protransform | proisagg | proiswindow | prosecdef | proleakproof | proisstrict | proretset | provolatile | pronargs | pronargdefaults | prorettype | proargtypes | proallargtypes | proargmodes | proargnames | proargdefaults | prosrc | probin | proconfig | proacl | prodefaultargpos | fencedmode | proshippable | propackage | prokind | proargsrc | propackageid | proisprivate | proargtypesext | prodefaultargposext | allargtypes | allargtypesext +-------+----------------------+--------------+----------+---------+---------+---------+-------------+--------------+----------+-------------+-----------+--------------+-------------+-----------+-------------+----------+-----------------+------------+--------------------+----------------+-------------+-------------+----------------+----------------------+--------------------------+-----------+--------+------------------+------------+--------------+------------+---------+-----------+--------------+--------------+----------------+---------------------+--------------------+---------------- +--? .* | gb18030_2022_to_utf8 | 11 | 10 | 13 | 1 | 0 | 0 | - | f | f | f | f | t | f | v | 5 | 0 | 2278 | 23 23 2275 2281 23 | | | | | gb18030_2022_to_utf8 | $libdir/utf8_and_gb18030 | | | | f | f | f | f | | 0 | f | | | 23 23 2275 2281 23 | +--? .* | gb18030_to_utf8 | 11 | 10 | 13 | 1 | 0 | 0 | - | f | f | f | f | t | f | v | 5 | 0 | 2278 | 23 23 2275 2281 23 | | | | | gb18030_to_utf8 | $libdir/utf8_and_gb18030 | | | | f | f | f | f | | 0 | f | | | 23 23 2275 2281 23 | +--? .* | utf8_to_gb18030 | 11 | 10 | 13 | 1 | 0 | 0 | - | f | f | f | f | t | f | v | 5 | 0 | 2278 | 23 23 2275 2281 23 | | | | | utf8_to_gb18030 | $libdir/utf8_and_gb18030 | | | | f | f | f | f | | 0 | f | | | 23 23 2275 2281 23 | +--? .* | utf8_to_gb18030_2022 | 11 | 10 | 13 | 1 | 0 | 0 | - | f | f | f | f | t | f | v | 5 | 0 | 2278 | 23 23 2275 2281 23 | | | | | utf8_to_gb18030_2022 | $libdir/utf8_and_gb18030 | | | | f | f | f | f | | 0 | f | | | 23 23 2275 2281 23 | +(4 rows) + + +--12. 看护client_encoding不能设置为GB18030_2022 +set client_encoding = GB18030_2022; +ERROR: invalid value for parameter "client_encoding": "gb18030_2022" + +ALTER SESSION SET NAMES 'GB18030_2022'; +ERROR: invalid value for parameter "client_encoding": "GB18030_2022" diff --git a/src/test/regress/parallel_schedule0 b/src/test/regress/parallel_schedule0 index d3135fa2d..90cdb60ca 100644 --- a/src/test/regress/parallel_schedule0 +++ b/src/test/regress/parallel_schedule0 @@ -1099,3 +1099,4 @@ test: enable_expr_fusion_flatten # test for on update timestamp and generated column test: on_update_session1 on_update_session2 +test: ts_gb18030_utf8 \ No newline at end of file diff --git a/src/test/regress/sql/ts_gb18030_utf8.sql b/src/test/regress/sql/ts_gb18030_utf8.sql deleted file mode 100644 index 1b14f2769..000000000 --- a/src/test/regress/sql/ts_gb18030_utf8.sql +++ /dev/null @@ -1,20 +0,0 @@ -create database gb18030 encoding='gb18030' LC_COLLATE='zh_CN.GB18030' LC_CTYPE ='zh_CN.GB18030' TEMPLATE=template0; -\c gb18030 - -show server_encoding; -create table tb_test(id int, content text); - -insert into tb_test values(1, 'abcdefghigkABCDEFGHIJK'); -insert into tb_test values(2, '12'); -insert into tb_test values(3, 'ĺ'); -insert into tb_test values(4, '019808'); -insert into tb_test values(5, '94 95 92 94 97 98 99 90'); -insert into tb_test values(5, '25'); - -select * from tb_test order by id; -select convert_to(content, 'utf8') from tb_test order by id; - -drop table tb_test; -\c regression -clean connection to all force for database gb18030; -drop database gb18030; From b91eddd95a7f0b2193869cc3e32644657ab7f1a9 Mon Sep 17 00:00:00 2001 From: openGaussDev Date: Mon, 28 Aug 2023 11:14:54 +0800 Subject: [PATCH 181/304] extreme_rto support standby read Offering: openGaussDev More detail: extreme_rto support standby read Match-id-bbd35285e701bafd9bbed15682fc20fc1449575e --- contrib/pagehack/pagehack.cpp | 271 +++++++- src/bin/gs_guc/cluster_guc.conf | 6 + src/bin/pg_rewind/fetch.cpp | 4 + src/common/backend/catalog/builtin_funcs.ini | 4 + src/common/backend/catalog/storage.cpp | 51 +- src/common/backend/utils/init/globals.cpp | 2 +- .../backend/utils/misc/guc/guc_storage.cpp | 81 +++ src/common/backend/utils/time/snapmgr.cpp | 3 + src/gausskernel/CMakeLists.txt | 1 + .../optimizer/commands/dbcommands.cpp | 7 +- .../optimizer/commands/vacuumlazy.cpp | 4 +- .../process/postmaster/postmaster.cpp | 65 +- .../process/threadpool/knl_instance.cpp | 3 +- .../process/threadpool/knl_thread.cpp | 13 + .../storage/access/redo/CMakeLists.txt | 1 + src/gausskernel/storage/access/redo/Makefile | 1 + .../storage/access/redo/redo_dbcommands.cpp | 1 + .../storage/access/redo/redo_storage.cpp | 2 +- .../access/redo/redo_visibilitymap.cpp | 1 + .../storage/access/redo/redo_xlogutils.cpp | 170 ++++- .../access/redo/standby_read/CMakeLists.txt | 23 + .../storage/access/redo/standby_read/Makefile | 37 + .../redo/standby_read/base_page_proc.cpp | 107 +++ .../redo/standby_read/block_info_proc.cpp | 299 ++++++++ .../standby_read/lsn_info_double_list.cpp | 74 ++ .../redo/standby_read/lsn_info_proc.cpp | 650 ++++++++++++++++++ .../standby_read/standby_read_interface.cpp | 351 ++++++++++ .../access/transam/extreme_rto/Makefile | 2 +- .../access/transam/extreme_rto/batch_redo.cpp | 19 - .../access/transam/extreme_rto/dispatcher.cpp | 45 +- .../transam/extreme_rto/exrto_recycle.cpp | 235 +++++++ .../access/transam/extreme_rto/page_redo.cpp | 555 +++++++++++---- .../storage/access/transam/multi_redo_api.cpp | 8 +- .../storage/access/transam/xact.cpp | 10 + .../storage/access/transam/xlog.cpp | 19 +- .../access/ustore/knl_uextremeredo.cpp | 41 +- .../access/ustore/undo/knl_uundoapi.cpp | 11 +- .../access/ustore/undo/knl_uundorecycle.cpp | 173 ++++- .../access/ustore/undo/knl_uundospace.cpp | 68 +- .../access/ustore/undo/knl_uundozone.cpp | 134 +++- src/gausskernel/storage/buffer/bufmgr.cpp | 83 ++- src/gausskernel/storage/ipc/procarray.cpp | 66 +- src/gausskernel/storage/lmgr/lwlocknames.txt | 1 + src/gausskernel/storage/lmgr/proc.cpp | 4 + src/gausskernel/storage/nvm/nvmbuffer.cpp | 6 +- src/gausskernel/storage/page/bufpage.cpp | 5 +- .../storage/replication/basebackup.cpp | 4 + src/gausskernel/storage/replication/slot.cpp | 2 + .../storage/replication/walreceiver.cpp | 2 + .../storage/replication/walsender.cpp | 2 + src/gausskernel/storage/smgr/Makefile | 2 +- src/gausskernel/storage/smgr/smgr.cpp | 47 +- .../storage/smgr/storage_exrto_file.cpp | 545 +++++++++++++++ src/include/access/extreme_rto/batch_redo.h | 20 + src/include/access/extreme_rto/dispatcher.h | 1 + src/include/access/extreme_rto/page_redo.h | 9 +- src/include/access/extreme_rto/standby_read.h | 30 + .../standby_read/block_info_meta.h | 105 +++ .../standby_read/lsn_info_double_list.h | 49 ++ .../extreme_rto/standby_read/lsn_info_meta.h | 151 ++++ .../standby_read/standby_read_base.h | 81 +++ src/include/access/multi_redo_api.h | 4 + .../access/ustore/undo/knl_uundospace.h | 12 + .../access/ustore/undo/knl_uundozone.h | 41 +- src/include/access/xlogproc.h | 8 +- src/include/catalog/storage.h | 4 +- src/include/catalog/storage_xlog.h | 6 +- .../rollback-post_catalog_maindb_92_909.sql | 1 + .../rollback-post_catalog_otherdb_92_909.sql | 1 + .../upgrade-post_catalog_maindb_92_909.sql | 11 + .../upgrade-post_catalog_otherdb_92_909.sql | 11 + src/include/gs_thread.h | 1 + .../knl/knl_guc/knl_instance_attr_storage.h | 8 + src/include/knl/knl_instance.h | 6 +- src/include/knl/knl_session.h | 6 + src/include/knl/knl_thread.h | 18 + src/include/miscadmin.h | 2 + src/include/postgres.h | 2 + src/include/postmaster/postmaster.h | 3 + src/include/replication/walreceiver.h | 3 + src/include/storage/buf/buf_internals.h | 18 + src/include/storage/buf/bufmgr.h | 10 + src/include/storage/buf/bufpage.h | 1 + src/include/storage/nvm/nvm.h | 2 +- src/include/storage/proc.h | 2 + src/include/storage/procarray.h | 2 + src/include/storage/smgr/relfilenode.h | 2 +- src/include/storage/smgr/smgr.h | 22 +- src/include/utils/snapshot.h | 26 + .../regress/output/recovery_2pc_tools.source | 6 + 90 files changed, 4731 insertions(+), 275 deletions(-) create mode 100644 src/gausskernel/storage/access/redo/standby_read/CMakeLists.txt create mode 100644 src/gausskernel/storage/access/redo/standby_read/Makefile create mode 100644 src/gausskernel/storage/access/redo/standby_read/base_page_proc.cpp create mode 100644 src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp create mode 100644 src/gausskernel/storage/access/redo/standby_read/lsn_info_double_list.cpp create mode 100644 src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp create mode 100644 src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp create mode 100644 src/gausskernel/storage/access/transam/extreme_rto/exrto_recycle.cpp create mode 100644 src/gausskernel/storage/smgr/storage_exrto_file.cpp create mode 100644 src/include/access/extreme_rto/standby_read.h create mode 100644 src/include/access/extreme_rto/standby_read/block_info_meta.h create mode 100644 src/include/access/extreme_rto/standby_read/lsn_info_double_list.h create mode 100644 src/include/access/extreme_rto/standby_read/lsn_info_meta.h create mode 100644 src/include/access/extreme_rto/standby_read/standby_read_base.h create mode 100644 src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback-post_catalog_maindb_92_909.sql create mode 100644 src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback-post_catalog_otherdb_92_909.sql create mode 100644 src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_909.sql create mode 100644 src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_909.sql diff --git a/contrib/pagehack/pagehack.cpp b/contrib/pagehack/pagehack.cpp index 8adaed889..9c527326f 100644 --- a/contrib/pagehack/pagehack.cpp +++ b/contrib/pagehack/pagehack.cpp @@ -60,6 +60,8 @@ #include "access/ustore/knl_utuple.h" #include "access/ustore/knl_uundorecord.h" #include "access/double_write_basic.h" +#include "access/extreme_rto/standby_read/block_info_meta.h" +#include "access/extreme_rto/standby_read/lsn_info_meta.h" #include "catalog/pg_control.h" #include "catalog/pg_attribute.h" #include "catalog/pg_class.h" @@ -102,7 +104,11 @@ /* Number of pg_class types */ #define CLASS_TYPE_NUM 512 #define TEN 10 - +#define BLOCK_META_INFO_NUM_PER_PAGE 127 +#define BASE_PAGE_MAP_SIZE 16 +#define BASE_PAGE_MAP_BIT_SIZE (BASE_PAGE_MAP_SIZE * BITS_PER_BYTE) +#define DIVIDED_BY_TWO 2 +#define WAL_ID_OFFSET 32 typedef unsigned char* binary; static const char* indents[] = { // 10 tab is enough to used. @@ -830,7 +836,9 @@ typedef enum HackingType { HACKING_UNDO_RECORD, HACKING_UNDO_FIX, HACKING_SEGMENT, - NUM_HACKINGTYPE + NUM_HACKINGTYPE, + HACKING_LSN_INFO_META, + HACKING_BLOCK_INFO_META, } HackingType; static HackingType hackingtype = HACKING_HEAP; @@ -856,7 +864,9 @@ static const char* HACKINGTYPE[] = {"heap", "undo_slot", "undo_record", "undo_fix", - "segment" + "segment", + "lsn_info_meta", + "block_info_meta" }; const char* PageTypeNames[] = {"DATA", "FSM", "VM"}; @@ -912,6 +922,7 @@ typedef struct FSMAddress { const int FSM_BOTTOM_LEVEL = 0; using namespace undo; +using namespace extreme_rto_standby_read; static void formatBytes(unsigned char* start, int len) { @@ -5036,6 +5047,248 @@ static int ParseUndoSlot(const char *filename) return true; } +static void parse_map_position(uint8 map) +{ + uint8 pagemap[BITS_PER_BYTE] = { 0 }; + int pos = 0; + pos = 0; + while (map > 0) { + pagemap[pos] = map % DIVIDED_BY_TWO; + ++pos; + map /= DIVIDED_BY_TWO; + } + for (int loop = BITS_PER_BYTE - 1; loop >= 0; loop--) { + fprintf(stdout, "%u", pagemap[loop]); + } + fprintf(stdout, " "); +} + +static void parse_lsn_info_head(LsnInfoPageHeader *header) +{ + PageXLogRecPtr lsn = header->lsn; + fprintf(stdout, "%slsn: xlogid %u, xrecoff %u, lsn %lu\n", + indents[indentLevel], lsn.xlogid, lsn.xrecoff, ((uint64)lsn.xlogid << WAL_ID_OFFSET) | lsn.xrecoff); + fprintf(stdout, "%schecksum: %u, flags: %u, version: %u", + indents[indentLevel], header->checksum, header->flags, header->version); + fprintf(stdout, "%sbase page map: ", indents[indentLevel]); + for (uint32 loop = 0; loop < BASE_PAGE_MAP_SIZE; loop++) { + parse_map_position(header->base_page_map[loop]); + } + fprintf(stdout, "\n"); +} + +static void parse_lsn_info_node(LsnInfoNode *lsninfo) +{ + fprintf(stdout, "%slsn info list: prev %lu, next: %lu\n", + indents[indentLevel], lsninfo->lsn_list.prev, lsninfo->lsn_list.next); + fprintf(stdout, "%sflags: %u, type: %u, used: %u\n", + indents[indentLevel], lsninfo->flags, lsninfo->type, lsninfo->used); + fprintf(stdout, "%slsn:", indents[indentLevel]); + for (uint loop = 0; loop < LSN_NUM_PER_NODE; loop++) { + fprintf(stdout, " %lu", lsninfo->lsn[loop]); + } + fprintf(stdout, "\n"); +} + +static void parse_base_page_info_node(BasePageInfoNode *pageinfo) +{ + RelFileNode rnode = pageinfo->relfilenode; + fprintf(stdout, "%slsn info:\n", indents[indentLevel]); + + ++indentLevel; + parse_lsn_info_node(&(pageinfo->lsn_info_node)); + --indentLevel; + + fprintf(stdout, "%sbase page list: prev %lu, next: %lu\n", + indents[indentLevel], pageinfo->base_page_list.prev, pageinfo->base_page_list.next); + fprintf(stdout, "%scurrent page lsn: %lu\n", + indents[indentLevel], pageinfo->cur_page_lsn); + fprintf(stdout, "%srefile node:\n", indents[indentLevel]); + ++indentLevel; + fprintf(stdout, "%sspcnode: %u, dbnode: %u, relnode: %u, bucketnode: %d, opt: %u\n", + indents[indentLevel], rnode.spcNode, rnode.dbNode, rnode.relNode, rnode.bucketNode, rnode.opt); + --indentLevel; + fprintf(stdout, "%sfork num: %d, block num: %u\n", + indents[indentLevel], pageinfo->fork_num, pageinfo->block_num); + fprintf(stdout, "%snext base page lsn: %lu, base page position: %lu\n", + indents[indentLevel], pageinfo->next_base_page_lsn, pageinfo->base_page_position); +} + +static void parse_lsn_info_block(FILE* fd, uint8 isbasepage[], uint32 &handledblock, uint32 loop) +{ + char bufferlsn[sizeof(LsnInfoNode)]; + char bufferpage[sizeof(BasePageInfoNode)]; + LsnInfoNode *lsnInfo = NULL; + BasePageInfoNode *basepageinfo = NULL; + + if (isbasepage[handledblock]) { + fprintf(stdout, "it's a basepage.\n"); + (void)fread(bufferpage, 1, sizeof(BasePageInfoNode), fd); + basepageinfo = (BasePageInfoNode *)bufferpage; + if (basepageinfo->lsn_info_node.type != LSN_INFO_TYPE_BASE_PAGE) { + fprintf(stderr, "Data at page %u, block %u must be base page, but its type is: %u.\n", + loop, handledblock, basepageinfo->lsn_info_node.type); // report error but continue. + } + parse_base_page_info_node(basepageinfo); + handledblock += 2; // index need add by 2 for basepage takes 2 blocks. + } else { + (void)fread(bufferlsn, 1, sizeof(LsnInfoNode), fd); + lsnInfo = (LsnInfoNode *)bufferlsn; + if (!is_lsn_info_node_valid(lsnInfo->flags)) { + fprintf(stdout, "Data at page %u, block %u is not valid.\n", loop, handledblock); + } else { + fprintf(stdout, "it's a lsn page.\n"); + if (lsnInfo->type != LSN_INFO_TYPE_LSNS) { + fprintf(stderr, "Data at page %u, block %u must be lsn page, but its type is: %u.\n", + loop, handledblock, lsnInfo->type); // report error but continue. + } + parse_lsn_info_node(lsnInfo); + } + handledblock++; + } +} + +static bool parse_lsn_info_meta(const char *filename) +{ + char bufferhead[sizeof(LsnInfoPageHeader)]; + LsnInfoPageHeader *pageheader = NULL; + FILE* fd = NULL; + uint32 loop, loopmap, loopbit, handledblock; + uint8 pagemappos; + uint8 isbasepage[BASE_PAGE_MAP_BIT_SIZE] = { 0 }; + if (NULL == (fd = fopen(filename, "rb"))) { + fprintf(stderr, "%s: %s\n", filename, strerror(errno)); + return false; + } + + fseek(fd, 0, SEEK_END); + long size = ftell(fd); + rewind(fd); + + if (size % BLCKSZ != 0) { + fprintf(stderr, "Reading lsn/page info meta file error: file size is not divisible by page size(8k).\n"); + fclose(fd); + return false; + } + + long pagenum = size / BLCKSZ; + fprintf(stdout, "file length is %ld, blknum is %ld\n", size, pagenum); + + for (loop = 1; loop <= pagenum; loop++) { + fprintf(stdout, "Page %u information:\n", loop); + ++indentLevel; + if (fread(bufferhead, 1, sizeof(LsnInfoPageHeader), fd) != sizeof(LsnInfoPageHeader)) { + fprintf(stderr, "%sReading header error", indents[indentLevel]); + fclose(fd); + return false; + } + + pageheader = (LsnInfoPageHeader *)bufferhead; + if (!is_lsn_info_page_valid(pageheader)) { + fseek(fd, (BASE_PAGE_MAP_BIT_SIZE - 1) * BLCKSZ, SEEK_SET); // push 127 * 64 bytes + fprintf(stdout, "%sPage %u is not valid.\n", indents[indentLevel], loop); + --indentLevel; + continue; + } + parse_lsn_info_head(pageheader); + + pagemappos = 0; + for (loopmap = 0; loopmap < BASE_PAGE_MAP_SIZE; loopmap++) { + for (loopbit = 0; loopbit < BITS_PER_BYTE; loopbit++) { + isbasepage[pagemappos] = (((pageheader->base_page_map[loopmap]) & (0x1 << loopbit)) >> loopbit); + pagemappos++; + } + } + + handledblock = 1; // 1st block is handled as header + while (handledblock < BASE_PAGE_MAP_BIT_SIZE) { + fprintf(stdout, "%sBlock %u information: ", indents[indentLevel], handledblock); + ++indentLevel; + parse_lsn_info_block(fd, isbasepage, handledblock, loop); + --indentLevel; + } + memset_s(isbasepage, sizeof(isbasepage), 0, sizeof(isbasepage)); + --indentLevel; + } + fclose(fd); + return true; +} + +static void parse_block_info_head(BlockInfoPageHeader *header) +{ + PageXLogRecPtr lsn = header->lsn; + fprintf(stdout, "%slsn: xlogid %u, xrecoff %u, lsn %lu\n", + indents[indentLevel], lsn.xlogid, lsn.xrecoff, ((uint64)lsn.xlogid << WAL_ID_OFFSET) | lsn.xrecoff); + fprintf(stdout, "%schecksum: %u, flags: %u\n", + indents[indentLevel], header->checksum, header->flags); + fprintf(stdout, "%sversion: %u, total_block_num: %lu\n", + indents[indentLevel], header->version, header->total_block_num); +} + +static void parse_block_info_content(BlockMetaInfo *blockInfo) +{ + fprintf(stdout, "%stimeline: %u, record_num: %u\n", + indents[indentLevel], blockInfo->timeline, blockInfo->record_num); + fprintf(stdout, "%smin_lsn: %lu, max_lsn: %lu, flags: %u\n", + indents[indentLevel], blockInfo->min_lsn, blockInfo->max_lsn, blockInfo->flags); + fprintf(stdout, "%slsn_info_list: prev %lu, next: %lu\n", + indents[indentLevel], blockInfo->lsn_info_list.prev, blockInfo->lsn_info_list.next); +} + +static bool parse_block_info_meta(const char *filename) +{ + char bufferhead[sizeof(BlockInfoPageHeader)]; + char bufferblock[sizeof(BlockMetaInfo)]; + uint32 loop, loopinfo; + FILE* fd = NULL; + + if (NULL == (fd = fopen(filename, "rb"))) { + fprintf(stderr, "%s: %s\n", filename, strerror(errno)); + return false; + } + + fseek(fd, 0, SEEK_END); + long size = ftell(fd); + rewind(fd); + + if (size % BLCKSZ != 0) { + fprintf(stderr, "Reading block info meta file error: file size is not divisible by page size(8k).\n"); + fclose(fd); + return false; + } + long pagenum = size / BLCKSZ; + fprintf(stdout, "file length is %ld, blknum is %ld\n", size, pagenum); + + for (loop = 0; loop < pagenum; loop++) { + fprintf(stdout, "Page %u information:\n", loop); + ++indentLevel; + + if (fread(bufferhead, 1, sizeof(BlockInfoPageHeader), fd) != sizeof(BlockInfoPageHeader)) { + fprintf(stderr, "%sReading header error", indents[indentLevel]); + fclose(fd); + return false; + } + parse_block_info_head((BlockInfoPageHeader *)bufferhead); + + for (loopinfo = 0; loopinfo < BLOCK_META_INFO_NUM_PER_PAGE; loopinfo++) { + fprintf(stdout, "%sBlock %u information:\n", indents[indentLevel], loopinfo); + ++indentLevel; + if (fread(bufferblock, 1, sizeof(BlockMetaInfo), fd) != sizeof(BlockMetaInfo)) { + fprintf(stderr, "%sReading block meta file error at %u page, %u block.\n", + indents[indentLevel], loop, loopinfo); + fclose(fd); + return false; + } + parse_block_info_content((BlockMetaInfo *)bufferblock); + --indentLevel; + } + --indentLevel; + } + + fclose(fd); + return true; +} + typedef struct UndoHeader { UndoRecordHeader whdr_; UndoRecordBlock wblk_; @@ -5956,6 +6209,18 @@ int main(int argc, char** argv) break; case HACKING_UNDO_FIX: break; + case HACKING_LSN_INFO_META: + if (!parse_lsn_info_meta(filename)) { + fprintf(stderr, "Error during parsing lsn info meta file %s\n", filename); + exit(1); + } + break; + case HACKING_BLOCK_INFO_META: + if (!parse_block_info_meta(filename)) { + fprintf(stderr, "Error during parsing block info meta file %s\n", filename); + exit(1); + } + break; default: /* should be impossible to be here */ Assert(false); diff --git a/src/bin/gs_guc/cluster_guc.conf b/src/bin/gs_guc/cluster_guc.conf index 321353905..bd587bef8 100755 --- a/src/bin/gs_guc/cluster_guc.conf +++ b/src/bin/gs_guc/cluster_guc.conf @@ -104,6 +104,7 @@ dirty_page_percent_max|real|0.1,1|NULL|NULL| group_concat_max_len|int64|0,9223372036854775807|NULL|NULL check_function_bodies|bool|0,0|NULL|NULL| checkpoint_completion_target|real|0,1|NULL|NULL| +standby_force_recyle_ratio|real|0,1|NULL|NULL| checkpoint_segments|int|1,2147483646|NULL|NULL| checkpoint_timeout|int|30,3600|s|NULL| checkpoint_warning|int|0,2147483647|s|NULL| @@ -268,6 +269,9 @@ geqo_effort|int|1,10|NULL|NULL| geqo_generations|int|0,2147483647|NULL|NULL| hadr_max_size_for_xlog_receiver|int|0,2147483647|kB|NULL| hadr_recovery_time_target|int|0,3600|NULL|NULL| +standby_recycle_interval|int|0,86400|s|NULL| +standby_max_query_time|int|0,86400|s|NULL| +base_page_saved_interval|int|4,2000|NULL|NULL| hadr_recovery_point_target|int|0,3600|NULL|NULL| hadr_super_user_record_path|string|0,0|NULL|NULL| hll_default_log2m|int|10,16|NULL|NULL| @@ -708,6 +712,8 @@ undo_zone_count|int|0,1048576|NULL|NULL| stream_cluster_run_mode|enum|cluster_primary,cluster_standby|NULL|NULL| xlog_file_size|int64|1048576,576460752303423487|B|The value must be an integer multiple of 16777216(16M)| xlog_file_path|string|0,0|NULL|NULL| +max_standby_base_page_size|int64|0,576460752303423487|B|NULL| +max_standby_lsn_info_size|int64|0,576460752303423487|B|NULL| plsql_show_all_error|bool|0,0|NULL|NULL| partition_page_estimation|bool|0,0|NULL|NULL| enable_auto_clean_unique_sql|bool|0,0|NULL|NULL| diff --git a/src/bin/pg_rewind/fetch.cpp b/src/bin/pg_rewind/fetch.cpp index 19d818eba..e1279943a 100755 --- a/src/bin/pg_rewind/fetch.cpp +++ b/src/bin/pg_rewind/fetch.cpp @@ -27,6 +27,7 @@ #include "PageCompression.h" #include "catalog/pg_type.h" #include "storage/file/fio_device.h" +#include "access/extreme_rto/standby_read/standby_read_base.h" PGconn* conn = NULL; char source_slot_name[NAMEDATALEN] = {0}; @@ -303,6 +304,9 @@ BuildErrorCode fetchSourceFileList() continue; if (NULL != strstr(path, "disable_conn_file")) continue; + if (NULL != strstr(path, EXRTO_FILE_DIR)) { + continue; + } if (PQgetisnull(res, 0, 1)) { /* diff --git a/src/common/backend/catalog/builtin_funcs.ini b/src/common/backend/catalog/builtin_funcs.ini index 0c59329f6..77c2dc72f 100755 --- a/src/common/backend/catalog/builtin_funcs.ini +++ b/src/common/backend/catalog/builtin_funcs.ini @@ -12904,3 +12904,7 @@ AddFuncGroup( "gs_repair_file", 1, AddBuiltinFunc(_0(4771), _1("gs_repair_file"), _2(3), _3(true), _4(true), _5(gs_repair_file), _6(16), _7(PG_CATALOG_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(1000), _12(0), _13(0), _14(false), _15(false), _16(false), _17(false), _18('s'), _19(0), _20(3, 26, 25, 23), _21(3, 26, 25, 23), _22(3, 'i', 'i', 'i'), _23(3, "tableoid", "path", "timeout"), _24(NULL), _25("gs_repair_file"), _26(NULL), _27(NULL), _28(NULL), _29(0), _30(false), _31(false), _32(false), _33(NULL), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)) ), + AddFuncGroup( + "gs_hot_standby_space_info", 1, + AddBuiltinFunc(_0(6218), _1("gs_hot_standby_space_info"), _2(0), _3(false), _4(true), _5(gs_hot_standby_space_info), _6(2249), _7(PG_CATALOG_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(0), _12(0), _13(0), _14(false), _15(false), _16(false), _17(false), _18('s'), _19(0), _20(0), _21(6, 28, 28, 28, 28, 28, 28), _22(6, 'o', 'o', 'o', 'o', 'o', 'o'), _23(6, "base_page_file_num", "base_page_total_size", "lsn_info_meta_file_num", "lsn_info_meta_total_size", "block_info_meta_file_num", "block_info_meta_total_size"), _24(NULL), _25("gs_hot_standby_space_info"), _26(NULL), _27(NULL), _28(NULL), _29(0), _30(false), _31(NULL), _32(false), _33(NULL), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)) + ), diff --git a/src/common/backend/catalog/storage.cpp b/src/common/backend/catalog/storage.cpp index 67e231c5f..6d6d559fb 100644 --- a/src/common/backend/catalog/storage.cpp +++ b/src/common/backend/catalog/storage.cpp @@ -24,6 +24,7 @@ #include "access/cstore_am.h" #include "access/visibilitymap.h" +#include "access/multi_redo_api.h" #include "access/xact.h" #include "access/xlog.h" #include "access/xloginsert.h" @@ -42,6 +43,7 @@ #include "pgxc/pgxc.h" #include "storage/freespace.h" #include "storage/lmgr.h" +#include "storage/procarray.h" #include "storage/smgr/smgr.h" #include "storage/smgr/segment.h" #include "threadpool/threadpool.h" @@ -608,7 +610,7 @@ void RelationPreserveStorage(RelFileNode rnode, bool atCommit) * This includes getting rid of any buffers for the blocks that are to be * dropped. */ -void RelationTruncate(Relation rel, BlockNumber nblocks) +void RelationTruncate(Relation rel, BlockNumber nblocks, TransactionId latest_removed_xid) { /* Currently, segment-page tables should not be truncated */ Assert(!RelationIsSegmentTable(rel)); @@ -675,14 +677,13 @@ void RelationTruncate(Relation rel, BlockNumber nblocks) uint size; uint8 info = XLOG_SMGR_TRUNCATE | XLR_SPECIAL_REL_UPDATE; + size = sizeof(xl_smgr_truncate_compress); xlrec.xlrec.blkno = nblocks; + xlrec.pageCompressOpts = rel->rd_node.opt; + xlrec.latest_removed_xid = latest_removed_xid; if (rel->rd_node.opt != 0) { - xlrec.pageCompressOpts = rel->rd_node.opt; - size = sizeof(xl_smgr_truncate_compress); info |= XLR_REL_COMPRESS; - } else { - size = sizeof(xl_smgr_truncate); } RelFileNodeRelCopy(xlrec.xlrec.rnode, rel->rd_node); @@ -713,7 +714,7 @@ void RelationTruncate(Relation rel, BlockNumber nblocks) BatchClearBadBlock(rel->rd_node, MAIN_FORKNUM, nblocks); } -void PartitionTruncate(Relation parent, Partition part, BlockNumber nblocks) +void PartitionTruncate(Relation parent, Partition part, BlockNumber nblocks, TransactionId latest_removed_xid) { /* Currently, segment-page tables should not be truncated */ Assert(!RelationIsSegmentTable(parent)); @@ -764,14 +765,16 @@ void PartitionTruncate(Relation parent, Partition part, BlockNumber nblocks) uint8 info = XLOG_SMGR_TRUNCATE | XLR_SPECIAL_REL_UPDATE; int redoSize; + redoSize = sizeof(xl_smgr_truncate_compress); + + xlrec.xlrec.blkno = nblocks; + xlrec.pageCompressOpts = rel->rd_node.opt; + xlrec.latest_removed_xid = latest_removed_xid; + if (rel->rd_node.opt != 0) { - xlrec.pageCompressOpts = rel->rd_node.opt; info |= XLR_REL_COMPRESS; - redoSize = sizeof(xl_smgr_truncate_compress); - } else { - redoSize = sizeof(xl_smgr_truncate); } - xlrec.xlrec.blkno = nblocks; + RelFileNodeRelCopy(xlrec.xlrec.rnode, part->pd_node); XLogBeginInsert(); @@ -1242,8 +1245,26 @@ void smgr_redo_create(RelFileNode rnode, ForkNumber forkNum, char *data) } } -void xlog_block_smgr_redo_truncate(RelFileNode rnode, BlockNumber blkno, XLogRecPtr lsn) +void smgr_redo_truncate_cancel_conflicting_proc(TransactionId latest_removed_xid) { + if (IS_EXRTO_READ) { + const int max_check_times = 1000; + int check_times = 0; + bool conflict = true; + bool reach_max_check_times = false; + while (conflict && check_times < max_check_times) { + RedoInterruptCallBack(); + check_times++; + reach_max_check_times = (check_times == max_check_times); + conflict = proc_array_cancel_conflicting_proc(latest_removed_xid, reach_max_check_times); + } + } +} + +void xlog_block_smgr_redo_truncate(RelFileNode rnode, BlockNumber blkno, XLogRecPtr lsn, + TransactionId latest_removed_xid) +{ + smgr_redo_truncate_cancel_conflicting_proc(latest_removed_xid); SMgrRelation reln = smgropen(rnode, InvalidBackendId); smgrcreate(reln, MAIN_FORKNUM, true); UpdateMinRecoveryPoint(lsn, false); @@ -1264,6 +1285,7 @@ void smgr_redo(XLogReaderState* record) XLogRecPtr lsn = record->EndRecPtr; uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; bool compress = (bool)(XLogRecGetInfo(record) & XLR_REL_COMPRESS); + TransactionId latest_removed_xid = InvalidTransactionId; /* Backup blocks are not used in smgr records */ Assert(!XLogRecHasAnyBlockRefs(record)); @@ -1280,6 +1302,9 @@ void smgr_redo(XLogReaderState* record) RelFileNode rnode; RelFileNodeCopy(rnode, xlrec->rnode, (int2)XLogRecGetBucketId(record)); rnode.opt = compress ? ((xl_smgr_truncate_compress*)(void *)XLogRecGetData(record))->pageCompressOpts : 0; + if (XLogRecGetDataLen(record) == TRUNCATE_CONTAIN_XID_SIZE) { + latest_removed_xid = ((xl_smgr_truncate_compress*)(void *)XLogRecGetData(record))->latest_removed_xid; + } /* * Forcibly create relation if it doesn't exist (which suggests that * it was dropped somewhere later in the WAL sequence). As in @@ -1305,7 +1330,7 @@ void smgr_redo(XLogReaderState* record) */ /* Also tell xlogutils.c about it */ - xlog_block_smgr_redo_truncate(rnode, xlrec->blkno, lsn); + xlog_block_smgr_redo_truncate(rnode, xlrec->blkno, lsn, latest_removed_xid); } else ereport(PANIC, (errmsg("smgr_redo: unknown op code %u", info))); } diff --git a/src/common/backend/utils/init/globals.cpp b/src/common/backend/utils/init/globals.cpp index d7824bfa3..1dd115185 100644 --- a/src/common/backend/utils/init/globals.cpp +++ b/src/common/backend/utils/init/globals.cpp @@ -75,7 +75,7 @@ bool will_shutdown = false; * NEXT | 92899 | ? | ? * ********************************************/ -const uint32 GRAND_VERSION_NUM = 92908; +const uint32 GRAND_VERSION_NUM = 92909; /******************************************** * 2.VERSION NUM FOR EACH FEATURE diff --git a/src/common/backend/utils/misc/guc/guc_storage.cpp b/src/common/backend/utils/misc/guc/guc_storage.cpp index ca3561f28..f3e3be89c 100755 --- a/src/common/backend/utils/misc/guc/guc_storage.cpp +++ b/src/common/backend/utils/misc/guc/guc_storage.cpp @@ -3263,6 +3263,48 @@ static void InitStorageConfigureNamesInt() NULL, NULL, NULL}, + {{"standby_recycle_interval", + PGC_SIGHUP, + NODE_ALL, + RESOURCES_RECOVERY, + gettext_noop("Sets the maximum wait time to recycle."), + NULL, + GUC_UNIT_S}, + &g_instance.attr.attr_storage.standby_recycle_interval, + 10, /* 10s */ + 0, + 3600 * 24, /* 24hour */ + NULL, + NULL, + NULL}, + {{"standby_max_query_time", + PGC_SIGHUP, + NODE_ALL, + RESOURCES_RECOVERY, + gettext_noop("Sets the maximum time allowed for query on standby."), + NULL, + GUC_UNIT_S}, + &g_instance.attr.attr_storage.standby_max_query_time, + 600, /* 10min */ + 0, + 3600 * 24, /* 24hour */ + NULL, + NULL, + NULL}, + {{"base_page_saved_interval", + PGC_POSTMASTER, + NODE_ALL, + RESOURCES_RECOVERY, + gettext_noop("Save a base page every time the page redo as many xlogs as the parameter value."), + NULL, + 0}, + &g_instance.attr.attr_storage.base_page_saved_interval, + 400, + 5, + 2000, + NULL, + NULL, + NULL}, {{"force_promote", PGC_POSTMASTER, NODE_ALL, @@ -3891,6 +3933,19 @@ static void InitStorageConfigureNamesReal() NULL, NULL, NULL}, + {{"standby_force_recyle_ratio", + PGC_SIGHUP, + NODE_ALL, + RESOURCES_RECOVERY, + gettext_noop("Sets the ratio that triggers forced recycling in extreme-rto standby read."), + NULL}, + &g_instance.attr.attr_storage.standby_force_recyle_ratio, + 0.8, + 0.0, + 1.0, + NULL, + NULL, + NULL}, {{"bypass_dram", PGC_SIGHUP, NODE_ALL, @@ -4041,6 +4096,32 @@ static void InitStorageConfigureNamesInt64() NULL, NULL, NULL}, + {{"max_standby_base_page_size", + PGC_POSTMASTER, + NODE_ALL, + RESOURCES_RECOVERY, + gettext_noop("Sets the max size of base page files on standby"), + NULL}, + &g_instance.attr.attr_storage.max_standby_base_page_size, + INT64CONST(0x4000000000), /* 256GB */ + INT64CONST(0), + INT64CONST(0x7FFFFFFFFFFFFFF), + NULL, + NULL, + NULL}, + {{"max_standby_lsn_info_size", + PGC_POSTMASTER, + NODE_ALL, + RESOURCES_RECOVERY, + gettext_noop("Sets the max size of lsn info files on standby"), + NULL}, + &g_instance.attr.attr_storage.max_standby_lsn_info_size, + INT64CONST(0x4000000000), /* 256GB */ + INT64CONST(0), + INT64CONST(0x7FFFFFFFFFFFFFF), + NULL, + NULL, + NULL}, /* End-of-list marker */ {{NULL, (GucContext)0, diff --git a/src/common/backend/utils/time/snapmgr.cpp b/src/common/backend/utils/time/snapmgr.cpp index ca405605d..3a288e254 100644 --- a/src/common/backend/utils/time/snapmgr.cpp +++ b/src/common/backend/utils/time/snapmgr.cpp @@ -1062,6 +1062,9 @@ static void SnapshotResetXmin(void) t_thrd.proc->snapCSN = InvalidCommitSeqNo; t_thrd.pgxact->csn_min = InvalidCommitSeqNo; t_thrd.pgxact->csn_dr = InvalidCommitSeqNo; + + t_thrd.proc->exrto_read_lsn = 0; + t_thrd.proc->exrto_gen_snap_time = 0; } } diff --git a/src/gausskernel/CMakeLists.txt b/src/gausskernel/CMakeLists.txt index 29fa43a5c..21b9e9e44 100755 --- a/src/gausskernel/CMakeLists.txt +++ b/src/gausskernel/CMakeLists.txt @@ -179,6 +179,7 @@ list(APPEND gaussdb_objects $ $ $ + $ $ $ $ diff --git a/src/gausskernel/optimizer/commands/dbcommands.cpp b/src/gausskernel/optimizer/commands/dbcommands.cpp index 89f68b399..631a96836 100644 --- a/src/gausskernel/optimizer/commands/dbcommands.cpp +++ b/src/gausskernel/optimizer/commands/dbcommands.cpp @@ -33,6 +33,8 @@ #include "access/xloginsert.h" #include "access/xlogutils.h" #include "access/multixact.h" +#include "access/multi_redo_api.h" +#include "access/extreme_rto/standby_read/block_info_meta.h" #include "catalog/catalog.h" #include "catalog/dependency.h" #include "catalog/indexing.h" @@ -2440,7 +2442,10 @@ void do_db_drop(Oid dbId, Oid tbSpcId) if (!rmtree(dst_path, true)) { ereport(WARNING, (errmsg("some useless files may be left behind in old database directory \"%s\"", dst_path))); } - + if (IS_EXRTO_READ) { + /* remove file start with {db_id}_ */ + extreme_rto_standby_read::remove_block_meta_info_files_of_db(dbId); + } if (InHotStandby) { /* * Release locks prior to commit. XXX There is a race condition diff --git a/src/gausskernel/optimizer/commands/vacuumlazy.cpp b/src/gausskernel/optimizer/commands/vacuumlazy.cpp index 6593883fa..75792c5cd 100644 --- a/src/gausskernel/optimizer/commands/vacuumlazy.cpp +++ b/src/gausskernel/optimizer/commands/vacuumlazy.cpp @@ -2057,9 +2057,9 @@ lazy_truncate_heap(Relation onerel, VacuumStmt *vacstmt, LVRelStats *vacrelstats */ if (RelationIsPartition(onerel)) { Assert(vacstmt->onepart && vacstmt->onepartrel); - PartitionTruncate(vacstmt->onepartrel, vacstmt->onepart, new_rel_pages); + PartitionTruncate(vacstmt->onepartrel, vacstmt->onepart, new_rel_pages, vacrelstats->latestRemovedXid); } else { - RelationTruncate(onerel, new_rel_pages); + RelationTruncate(onerel, new_rel_pages, vacrelstats->latestRemovedXid); } /* diff --git a/src/gausskernel/process/postmaster/postmaster.cpp b/src/gausskernel/process/postmaster/postmaster.cpp index ea5d789a7..c85ca2f44 100644 --- a/src/gausskernel/process/postmaster/postmaster.cpp +++ b/src/gausskernel/process/postmaster/postmaster.cpp @@ -80,6 +80,7 @@ #endif #include "access/cbmparsexlog.h" +#include "access/extreme_rto/standby_read.h" #include "access/obs/obs_am.h" #include "access/transam.h" #include "access/ustore/undo/knl_uundoapi.h" @@ -233,6 +234,7 @@ #include "access/multi_redo_api.h" #include "postmaster/postmaster.h" #include "access/parallel_recovery/dispatcher.h" +#include "access/extreme_rto/standby_read/standby_read_base.h" #include "utils/distribute_test.h" #ifdef ENABLE_MULTIPLE_NODES #include "tsdb/compaction/compaction_entry.h" @@ -313,6 +315,8 @@ extern void gs_set_hs_shm_data(HaShmemData* ha_shm_data); extern void ReaperBackendMain(); extern void AdjustThreadAffinity(); +extern void exrto_standby_read_init(); + #define EXTERN_SLOTS_NUM 17 volatile PMState pmState = PM_INIT; bool dummyStandbyMode = false; @@ -3090,10 +3094,7 @@ int PostmasterMain(int argc, char* argv[]) /* init sharestorge(dorado) */ ShareStorageInit(); - - /* - * We're ready to rock and roll... - */ + exrto_standby_read_init(); if (ENABLE_DMS && ENABLE_REFORM) { if (!DMSWaitInitStartup()) { if (g_instance.pid_cxt.StartupPID == 0) { @@ -3347,10 +3348,10 @@ static void CheckExtremeRtoGUCConflicts(void) } #ifndef ENABLE_MULTIPLE_NODES - if ((g_instance.attr.attr_storage.recovery_parse_workers > 1) && g_instance.attr.attr_storage.EnableHotStandby) { + if (IS_DISASTER_RECOVER_MODE &&(g_instance.attr.attr_storage.recovery_parse_workers > 1) && g_instance.attr.attr_storage.EnableHotStandby) { ereport(ERROR, (errcode(ERRCODE_SYSTEM_ERROR), - errmsg("extreme rto could not support hot standby."), + errmsg("For disaster standby cluster, extreme rto could not support hot standby."), errhint("Either turn off extreme rto, or turn off hot_standby."))); } #endif @@ -4313,7 +4314,7 @@ static int ServerLoop(void) if (g_instance.attr.attr_storage.enable_ustore && g_instance.pid_cxt.GlobalStatsPID == 0 && - pmState == PM_RUN) { + (pmState == PM_RUN || pmState == PM_HOT_STANDBY)) { g_instance.pid_cxt.GlobalStatsPID = initialize_util_thread(GLOBALSTATS_THREAD); } @@ -5094,7 +5095,7 @@ int ProcessStartupPacket(Port* port, bool SSLdone) } else { #ifdef ENABLE_MULTIPLE_NODES if (STANDBY_MODE == hashmdata->current_mode && (!IS_MULTI_DISASTER_RECOVER_MODE || GTM_FREE_MODE || - g_instance.attr.attr_storage.recovery_parse_workers > 1)) { + (IS_PGXC_DATANODE && !g_instance.attr.attr_storage.EnableHotStandby))) { ereport(ERROR, (errcode(ERRCODE_CANNOT_CONNECT_NOW), errmsg("can not accept connection in standby mode."))); } @@ -5868,6 +5869,10 @@ static void SIGHUP_handler(SIGNAL_ARGS) signal_child(g_instance.pid_cxt.UndoRecyclerPID, SIGHUP); } + if (g_instance.pid_cxt.exrto_recycler_pid != 0) { + signal_child(g_instance.pid_cxt.exrto_recycler_pid, SIGHUP); + } + if (g_instance.pid_cxt.GlobalStatsPID != 0) { signal_child(g_instance.pid_cxt.GlobalStatsPID, SIGHUP); } @@ -6959,7 +6964,11 @@ static void reaper(SIGNAL_ARGS) if (g_instance.pid_cxt.CBMWriterPID == 0 && !dummyStandbyMode && u_sess->attr.attr_storage.enable_cbm_tracking) - g_instance.pid_cxt.CBMWriterPID = initialize_util_thread(CBMWRITER); + + + if (IS_EXRTO_READ && g_instance.pid_cxt.exrto_recycler_pid == 0) { + g_instance.pid_cxt.exrto_recycler_pid = initialize_util_thread(EXRTO_RECYCLER); + } /* * Likewise, start other special children as needed. In a restart @@ -7746,6 +7755,15 @@ static void reaper(SIGNAL_ARGS) continue; } + if (pid == g_instance.pid_cxt.exrto_recycler_pid) { + g_instance.pid_cxt.exrto_recycler_pid = 0; + + if (!EXIT_STATUS_0(exitstatus)) { + HandleChildCrash(pid, exitstatus, _("Exrto recycle process")); + } + continue; + } + if (get_real_recovery_parallelism() > 1) { PageRedoExitStatus pageredoStatus = CheckExitPageWorkers(pid); if (pageredoStatus == PAGE_REDO_THREAD_EXIT_NORMAL) { @@ -8328,6 +8346,7 @@ static void AsssertAllChildThreadExit() Assert(g_instance.pid_cxt.CommPoolerCleanPID == 0); Assert(g_instance.pid_cxt.UndoLauncherPID == 0); Assert(g_instance.pid_cxt.UndoRecyclerPID == 0); + Assert(g_instance.pid_cxt.exrto_recycler_pid == 0); #ifndef ENABLE_MULTIPLE_NODES Assert(g_instance.pid_cxt.ApplyLauncerPID == 0); #endif @@ -8401,7 +8420,7 @@ static void PostmasterStateMachine(void) #endif /* ENABLE_MULTIPLE_NODES */ g_instance.pid_cxt.UndoLauncherPID == 0 && g_instance.pid_cxt.UndoRecyclerPID == 0 && - g_instance.pid_cxt.GlobalStatsPID == 0 && + g_instance.pid_cxt.exrto_recycler_pid == 0 && g_instance.pid_cxt.GlobalStatsPID == 0 && #ifndef ENABLE_MULTIPLE_NODES g_instance.pid_cxt.ApplyLauncerPID == 0 && #endif @@ -8619,6 +8638,7 @@ static void PostmasterStateMachine(void) hashmdata = t_thrd.postmaster_cxt.HaShmData; hashmdata->current_mode = cur_mode; NotifyGscHotStandby(); + exrto_standby_read_init(); g_instance.pid_cxt.StartupPID = initialize_util_thread(STARTUP); Assert(g_instance.pid_cxt.StartupPID != 0); pmState = PM_STARTUP; @@ -8662,6 +8682,7 @@ static void PostmasterStateMachine(void) PMUpdateDBState(STARTING_STATE, get_cur_mode(), get_cur_repl_num()); } + exrto_standby_read_init(); g_instance.pid_cxt.StartupPID = initialize_util_thread(STARTUP); Assert(g_instance.pid_cxt.StartupPID != 0); pmState = PM_STARTUP; @@ -13197,6 +13218,21 @@ bool PMstateIsRun(void) return PM_RUN == pmState; } +bool pm_state_is_startup() +{ + return (pmState == PM_STARTUP); +} + +bool pm_state_is_recovery() +{ + return (pmState == PM_RECOVERY); +} + +bool pm_state_is_hot_standby() +{ + return (pmState == PM_HOT_STANDBY); +} + /* malloc api of cJSON at backend side */ static void* cJSON_internal_malloc(size_t size) { @@ -13296,6 +13332,9 @@ static void SetAuxType() case SHARE_STORAGE_XLOG_COPYER: t_thrd.bootstrap_cxt.MyAuxProcType = XlogCopyBackendProcess; break; + case EXRTO_RECYCLER: + t_thrd.bootstrap_cxt.MyAuxProcType = ExrtoRecyclerProcess; + break; #ifdef ENABLE_MULTIPLE_NODES case BARRIER_PREPARSE: t_thrd.bootstrap_cxt.MyAuxProcType = BarrierPreParseBackendProcess; @@ -13589,6 +13628,10 @@ int GaussDbAuxiliaryThreadMain(knl_thread_arg* arg) SharedStorageXlogCopyBackendMain(); proc_exit(1); break; + case EXRTO_RECYCLER: + extreme_rto::exrto_recycle_main(); + proc_exit(1); + break; #ifdef ENABLE_MULTIPLE_NODES case BARRIER_PREPARSE: BarrierPreParseMain(); @@ -13846,6 +13889,7 @@ int GaussDbThreadMain(knl_thread_arg* arg) case PAGEREPAIR_THREAD: case HEARTBEAT: case SHARE_STORAGE_XLOG_COPYER: + case EXRTO_RECYCLER: #ifdef ENABLE_MULTIPLE_NODES case BARRIER_PREPARSE: case TS_COMPACTION: @@ -14399,6 +14443,7 @@ static ThreadMetaData GaussdbThreadGate[] = { { GaussDbThreadMain, APPLY_WORKER, "applyworker", "apply worker" }, { GaussDbThreadMain, STACK_PERF_WORKER, "stack_perf", "stack perf worker" }, { GaussDbThreadMain, DMS_AUXILIARY_THREAD, "dms_auxiliary", "maintenance xmin in dms" }, + { GaussDbThreadMain, EXRTO_RECYCLER, "exrtorecycler", "exrto recycler" }, /* Keep the block in the end if it may be absent !!! */ #ifdef ENABLE_MULTIPLE_NODES diff --git a/src/gausskernel/process/threadpool/knl_instance.cpp b/src/gausskernel/process/threadpool/knl_instance.cpp index 216f6f21d..92ef81cb2 100755 --- a/src/gausskernel/process/threadpool/knl_instance.cpp +++ b/src/gausskernel/process/threadpool/knl_instance.cpp @@ -324,7 +324,7 @@ static void knl_g_parallel_redo_init(knl_g_parallel_redo_context* predo_cxt) rc = memset_s(&predo_cxt->redoCpuBindcontrl, sizeof(RedoCpuBindControl), 0, sizeof(RedoCpuBindControl)); securec_check(rc, "", ""); - + predo_cxt->global_recycle_lsn = InvalidXLogRecPtr; predo_cxt->redoItemHash = NULL; } @@ -501,6 +501,7 @@ static void KnlGUndoInit(knl_g_undo_context *undoCxt) undoCxt->undoChainTotalSize = 0; undoCxt->globalFrozenXid = InvalidTransactionId; undoCxt->globalRecycleXid = InvalidTransactionId; + undoCxt->is_exrto_residual_undo_file_recycled = false; } static void knl_g_flashback_init(knl_g_flashback_context *flashbackCxt) diff --git a/src/gausskernel/process/threadpool/knl_thread.cpp b/src/gausskernel/process/threadpool/knl_thread.cpp index c321ac6e1..3c26794b7 100755 --- a/src/gausskernel/process/threadpool/knl_thread.cpp +++ b/src/gausskernel/process/threadpool/knl_thread.cpp @@ -960,6 +960,15 @@ static void knl_t_page_redo_init(knl_t_page_redo_context* page_redo_cxt) page_redo_cxt->got_SIGHUP = false; page_redo_cxt->sleep_long = false; page_redo_cxt->check_repair = false; + page_redo_cxt->redo_worker_ptr = NULL; +} + +static void knl_t_exrto_recycle_init(knl_t_exrto_recycle_context* exrto_recycle_cxt) +{ + exrto_recycle_cxt->shutdown_requested = false; + exrto_recycle_cxt->got_SIGHUP = false; + exrto_recycle_cxt->lsn_info.lsn_num = 0; + exrto_recycle_cxt->lsn_info.lsn_array = NULL; } static void knl_t_parallel_decode_init(knl_t_parallel_decode_worker_context* parallel_decode_cxt) @@ -1315,7 +1324,9 @@ static void knl_t_storage_init(knl_t_storage_context* storage_cxt) storage_cxt->BackendWritebackContext = (WritebackContext*)palloc0(sizeof(WritebackContext)); storage_cxt->SharedBufHash = NULL; storage_cxt->InProgressBuf = NULL; + storage_cxt->ParentInProgressBuf = NULL; storage_cxt->IsForInput = false; + storage_cxt->ParentIsForInput = false; storage_cxt->PinCountWaitBuf = NULL; storage_cxt->InProgressAioDispatch = NULL; storage_cxt->InProgressAioDispatchCount = 0; @@ -1887,6 +1898,7 @@ void knl_thread_init(knl_thread_role role) knl_t_pencentile_init(&t_thrd.percentile_cxt); knl_t_perf_snap_init(&t_thrd.perf_snap_cxt); knl_t_page_redo_init(&t_thrd.page_redo_cxt); + knl_t_exrto_recycle_init(&t_thrd.exrto_recycle_cxt); knl_t_parallel_decode_init(&t_thrd.parallel_decode_cxt); knl_t_parallel_decode_reader_init(&t_thrd.logicalreadworker_cxt); knl_t_heartbeat_init(&t_thrd.heartbeat_cxt); @@ -1952,6 +1964,7 @@ void RedoInterruptCallBack() Assert(!AmStartupProcess()); Assert(!AmPageRedoWorker()); + Assert(!AmErosRecyclerProcess()); } void RedoPageRepairCallBack(RepairBlockKey key, XLogPhyBlock pblk) diff --git a/src/gausskernel/storage/access/redo/CMakeLists.txt b/src/gausskernel/storage/access/redo/CMakeLists.txt index 16abb90f3..22cecf9f2 100755 --- a/src/gausskernel/storage/access/redo/CMakeLists.txt +++ b/src/gausskernel/storage/access/redo/CMakeLists.txt @@ -1,4 +1,5 @@ #This is the main CMAKE for build bin. +add_subdirectory(standby_read) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} TGT_redo_SRC) set(TGT_redo_INC diff --git a/src/gausskernel/storage/access/redo/Makefile b/src/gausskernel/storage/access/redo/Makefile index 9ec819f8b..07f1f8ca6 100644 --- a/src/gausskernel/storage/access/redo/Makefile +++ b/src/gausskernel/storage/access/redo/Makefile @@ -22,6 +22,7 @@ #------------------------------------------------------------------------- subdir = src/gausskernel/storage/access/redo +SUBDIRS = standby_read top_builddir = ../../../../.. include $(top_builddir)/src/Makefile.global diff --git a/src/gausskernel/storage/access/redo/redo_dbcommands.cpp b/src/gausskernel/storage/access/redo/redo_dbcommands.cpp index bc8314f42..4ab0a3ac4 100644 --- a/src/gausskernel/storage/access/redo/redo_dbcommands.cpp +++ b/src/gausskernel/storage/access/redo/redo_dbcommands.cpp @@ -86,6 +86,7 @@ XLogRecParseState *DbaseRedoParseToBlock(XLogReaderState *record, uint32 *blockn if ((info == XLOG_DBASE_CREATE) || (info == XLOG_DBASE_DROP)) { recordstatehead = DatabaseXlogCommonParseToBlock(record, blocknum); + recordstatehead->isFullSync = record->isFullSync; } else { ereport(PANIC, (errcode(ERRCODE_UNRECOGNIZED_NODE_TYPE), errmsg("DbaseRedoParseToBlock: unknown op code %u", info))); diff --git a/src/gausskernel/storage/access/redo/redo_storage.cpp b/src/gausskernel/storage/access/redo/redo_storage.cpp index 35aa13e0a..73a25fbec 100644 --- a/src/gausskernel/storage/access/redo/redo_storage.cpp +++ b/src/gausskernel/storage/access/redo/redo_storage.cpp @@ -68,7 +68,7 @@ XLogRecParseState *smgr_xlog_relnode_parse_to_block(XLogReaderState *record, uin XLogRecSetBlockCommonState(record, BLOCK_DATA_DDL_TYPE, filenode, recordstatehead); XLogRecSetBlockDdlState(&(recordstatehead->blockparse.extra_rec.blockddlrec), ddltype, - (char *)XLogRecGetData(record), 1, compress); + (char *)XLogRecGetData(record), 1, compress, XLogRecGetDataLen(record)); return recordstatehead; } diff --git a/src/gausskernel/storage/access/redo/redo_visibilitymap.cpp b/src/gausskernel/storage/access/redo/redo_visibilitymap.cpp index 0f3c42883..753b99d7b 100644 --- a/src/gausskernel/storage/access/redo/redo_visibilitymap.cpp +++ b/src/gausskernel/storage/access/redo/redo_visibilitymap.cpp @@ -49,6 +49,7 @@ bool visibilitymap_clear_page(Page mapPage, BlockNumber heapBlk) void visibilitymap_clear_buffer(RedoBufferInfo *bufferInfo, BlockNumber heapBlk) { if (visibilitymap_clear_page(bufferInfo->pageinfo.page, heapBlk)) { + PageSetLSN(bufferInfo->pageinfo.page, bufferInfo->lsn, false); MakeRedoBufferDirty(bufferInfo); } } diff --git a/src/gausskernel/storage/access/redo/redo_xlogutils.cpp b/src/gausskernel/storage/access/redo/redo_xlogutils.cpp index 0fb813d5b..7bfcbaf22 100644 --- a/src/gausskernel/storage/access/redo/redo_xlogutils.cpp +++ b/src/gausskernel/storage/access/redo/redo_xlogutils.cpp @@ -50,6 +50,9 @@ #include "access/ustore/knl_uextremeredo.h" #include "commands/dbcommands.h" +#include "access/extreme_rto/standby_read/block_info_meta.h" +#include "access/extreme_rto/batch_redo.h" +#include "access/extreme_rto/page_redo.h" #include "access/twophase.h" #include "access/redo_common.h" #include "ddes/dms/ss_dms_bufmgr.h" @@ -805,11 +808,12 @@ void XLogUpdateCopyedBlockState(XLogRecParseState *recordblockstate, XLogBlockPa } void XLogRecSetBlockDdlState(XLogBlockDdlParse *blockddlstate, uint32 blockddltype, char *mainData, - int rels, bool compress) + int rels, bool compress, uint32 mainDataLen) { Assert(blockddlstate != NULL); blockddlstate->blockddltype = blockddltype; blockddlstate->rels = rels; + blockddlstate->mainDataLen = mainDataLen; blockddlstate->mainData = mainData; blockddlstate->compress = compress; } @@ -1495,9 +1499,14 @@ void XLogBlockDdlDoSmgrAction(XLogBlockHead *blockhead, void *blockrecbody, Redo case BLOCK_DDL_CREATE_RELNODE: smgr_redo_create(rnode, blockhead->forknum, blockddlrec->mainData); break; - case BLOCK_DDL_TRUNCATE_RELNODE: - xlog_block_smgr_redo_truncate(rnode, blockhead->blkno, blockhead->end_ptr); + case BLOCK_DDL_TRUNCATE_RELNODE: { + TransactionId latest_removed_xid = InvalidTransactionId; + if (blockddlrec->mainDataLen == TRUNCATE_CONTAIN_XID_SIZE) { + latest_removed_xid = ((xl_smgr_truncate_compress*)blockddlrec->mainData)->latest_removed_xid; + } + xlog_block_smgr_redo_truncate(rnode, blockhead->blkno, blockhead->end_ptr, latest_removed_xid); break; + } case BLOCK_DDL_DROP_RELNODE: { bool compress = blockddlrec->compress; ColFileNodeRel *xnodes = (ColFileNodeRel *)blockddlrec->mainData; @@ -1748,26 +1757,38 @@ bool XLogBlockRedoForExtremeRTO(XLogRecParseState *redoblocktate, RedoBufferInfo } bool checkvalid = XLogBlockRefreshRedoBufferInfo(blockhead, bufferinfo); - if (!checkvalid) { + if (unlikely(!checkvalid)) { ereport(PANIC, (errmsg("XLogBlockRedoForExtremeRTO: redobuffer checkfailed"))); } - if (block_valid <= BLOCK_DATA_FSM_TYPE) { - if (redoaction != BLK_DONE) { - GetRedoStartTime(redoCost); - Assert(block_valid == g_xlogExtRtoRedoTable[block_valid].block_valid); - g_xlogExtRtoRedoTable[block_valid].xlog_redoextrto(blockhead, blockrecbody, bufferinfo); - CountRedoTime(redoCost); - } -#ifdef USE_ASSERT_CHECKING - if (block_valid != BLOCK_DATA_UNDO_TYPE && !bufferinfo->pageinfo.ignorecheck) { - DoRecordCheck(redoblocktate, PageGetLSN(bufferinfo->pageinfo.page), true); - } -#endif - AddReadBlock(redoblocktate, (u_sess->instr_cxt.pg_buffer_usage->shared_blks_read - readcount)); - } else { + + if (unlikely(block_valid > BLOCK_DATA_FSM_TYPE)) { ereport(WARNING, (errmsg("XLogBlockRedoForExtremeRTO: unsuport type %u, lsn %X/%X", (uint32)block_valid, (uint32)(blockhead->end_ptr >> 32), (uint32)(blockhead->end_ptr)))); + return false; + } + + if ((block_valid != BLOCK_DATA_UNDO_TYPE) && g_instance.attr.attr_storage.EnableHotStandby && + XLByteLT(PageGetLSN(bufferinfo->pageinfo.page), blockhead->end_ptr)) { + BufferTag buf_tag; + INIT_BUFFERTAG(buf_tag, bufferinfo->blockinfo.rnode, + bufferinfo->blockinfo.forknum, bufferinfo->blockinfo.blkno); + extreme_rto_standby_read::insert_lsn_to_block_info(&extreme_rto::g_redoWorker->standby_read_meta_info, buf_tag, + bufferinfo->pageinfo.page, blockhead->start_ptr); + } + + if (redoaction != BLK_DONE) { + GetRedoStartTime(redoCost); + Assert(block_valid == g_xlogExtRtoRedoTable[block_valid].block_valid); + g_xlogExtRtoRedoTable[block_valid].xlog_redoextrto(blockhead, blockrecbody, bufferinfo); + CountRedoTime(redoCost); + } +#ifdef USE_ASSERT_CHECKING + if (block_valid != BLOCK_DATA_UNDO_TYPE && !bufferinfo->pageinfo.ignorecheck) { + DoRecordCheck(redoblocktate, PageGetLSN(bufferinfo->pageinfo.page), true); } +#endif + AddReadBlock(redoblocktate, (u_sess->instr_cxt.pg_buffer_usage->shared_blks_read - readcount)); + return false; } @@ -1865,6 +1886,119 @@ void XLogBlockDispatchForExtermeRTO(XLogRecParseState *recordblockstate) } while (nextstate != NULL); } +bool find_target_state(XLogRecParseState *state_iter, const RedoBufferTag &target_tag) +{ + RelFileNode n; + uint32 blk; + ForkNumber fork; + extreme_rto::PRXLogRecGetBlockTag(state_iter, &n, &blk, &fork); + if (RelFileNodeEquals(n, target_tag.rnode) && target_tag.blkno == blk && target_tag.forknum == fork) { + return true; + } else { + return false; + } +} + +void wal_block_redo_for_extreme_rto_read(XLogRecParseState *state, RedoBufferInfo *buf_info) +{ + uint16 block_valid; + void *block_rec_body; + XLogBlockHead *block_head; + const int shift_size = 32; + + /* decode blockdata body */ + block_head = &state->blockparse.blockhead; + block_rec_body = &state->blockparse.extra_rec; + block_valid = XLogBlockHeadGetValidInfo(block_head); + + bool check_valid = XLogBlockRefreshRedoBufferInfo(block_head, buf_info); + if (!check_valid) { + ereport(ERROR, (errmsg("wal_block_redo_for_extreme_rto: redobuffer checkfailed"))); + } + if (block_valid <= BLOCK_DATA_FSM_TYPE) { + Assert(block_valid == g_xlogExtRtoRedoTable[block_valid].block_valid); + g_xlogExtRtoRedoTable[block_valid].xlog_redoextrto(block_head, block_rec_body, buf_info); + } else { + ereport(ERROR, (errmsg("wal_block_redo_for_extreme_rto: unsuport type %u, lsn %X/%X", (uint32)block_valid, + (uint32)(block_head->end_ptr >> shift_size), (uint32)(block_head->end_ptr)))); + } +} + +void init_redo_buffer_info(RedoBufferInfo *rb_info, const BufferTag &buf_tag, Buffer buf) +{ + rb_info->lsn = InvalidXLogRecPtr; + rb_info->buf = buf; + rb_info->blockinfo.rnode = buf_tag.rnode; + rb_info->blockinfo.forknum = buf_tag.forkNum; + rb_info->blockinfo.blkno = buf_tag.blockNum; + rb_info->blockinfo.pblk.block = InvalidBlockNumber; + rb_info->blockinfo.pblk.lsn = InvalidXLogRecPtr; + rb_info->blockinfo.pblk.relNode = InvalidOid; + rb_info->pageinfo.page = BufferGetPage(buf); + rb_info->pageinfo.pagesize = BufferGetPageSize(buf); +#ifdef USE_ASSERT_CHECKING + rb_info->pageinfo.ignorecheck = false; /* initial value */ +#endif + rb_info->dirtyflag = false; /* initial value, actually, dirtyflag is useless in extreme RTO read */ +} + +void redo_target_page(const BufferTag &buf_tag, StandbyReadLsnInfoArray *lsn_info, Buffer base_page_buf) +{ + char *error_msg = NULL; + RedoParseManager redo_pm; + + XLogReaderState *xlog_reader = XLogReaderAllocate(&read_local_xlog_page, NULL); + /* do we need register interrupt func here? like ProcessConfigFile */ + XLogParseBufferInitFunc(&redo_pm, MAX_BUFFER_NUM_PER_WAL_RECORD, NULL, NULL); + if (xlog_reader == NULL) { + ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"), + errdetail("Failed while allocating an XLog reading processor."))); + } + + RedoBufferInfo buf_info; + init_redo_buffer_info(&buf_info, buf_tag, base_page_buf); + for (uint32 i = 0; i < lsn_info->lsn_num; i++) { + XLogRecord *record = XLogReadRecord(xlog_reader, lsn_info->lsn_array[i], &error_msg); + if (record == NULL) { + ereport(ERROR, (errcode_for_file_access(), + errmsg("could not read two-phase state from xlog at %X/%X, errormsg: %s", + (uint32)(lsn_info->lsn_array[i] >> LSN_MOVE32), (uint32)(lsn_info->lsn_array[i]), + error_msg ? error_msg : " "))); + } + + uint32 num = 0; + XLogRecParseState *state = XLogParseToBlockCommonFunc(xlog_reader, &num); + + if (num == 0) { + ereport(ERROR, (errmsg("internal error, xlog in lsn %X/%X doesn't contain any block.", + (uint32)(lsn_info->lsn_array[i] >> LSN_MOVE32), (uint32)(lsn_info->lsn_array[i])))); + } + + if (state == NULL) { + ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"), + errdetail("Failed while wal parse to block."))); + } + XLogRecParseState *state_iter = state; + while (state_iter != NULL) { + if (find_target_state(state_iter, buf_info.blockinfo)) { + break; + } + state_iter = (XLogRecParseState *)(state_iter->nextrecord); + } + if (state_iter == NULL) { + ereport(ERROR, (errmsg("internal error, xlog in lsn %X/%X doesn't contain target block.", + (uint32)(lsn_info->lsn_array[i] >> LSN_MOVE32), (uint32)(lsn_info->lsn_array[i])))); + } + buf_info.lsn = state_iter->blockparse.blockhead.end_ptr; + buf_info.blockinfo.pblk = state_iter->blockparse.blockhead.pblk; + wal_block_redo_for_extreme_rto_read(state_iter, &buf_info); + XLogBlockParseStateRelease(state); + } + + XLogReaderFree(xlog_reader); + XLogParseBufferDestoryFunc(&redo_pm); +} + #ifdef EXTREME_RTO_DEBUG_AB void DoThreadExit() { diff --git a/src/gausskernel/storage/access/redo/standby_read/CMakeLists.txt b/src/gausskernel/storage/access/redo/standby_read/CMakeLists.txt new file mode 100644 index 000000000..7f8959afa --- /dev/null +++ b/src/gausskernel/storage/access/redo/standby_read/CMakeLists.txt @@ -0,0 +1,23 @@ +#This is the main CMAKE for build bin. + + +AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} TGT_standby_read_SRC) + +set(TGT_standby_read_INC + ${PROJECT_SRC_DIR}/tools/log_fdw + ${PROJECT_TRUNK_DIR}/distribute/bin/gds + ${PROJECT_SRC_DIR}/include/iprange + ${PROJECT_SRC_DIR}/include/libcomm + ${PROJECT_SRC_DIR}/include + ${PROJECT_SRC_DIR}/lib/gstrace + ${LIBCGROUP_INCLUDE_PATH} + ${ZLIB_INCLUDE_PATH} + ${LIBCURL_INCLUDE_PATH} + ${LZ4_INCLUDE_PATH} + ${EVENT_INCLUDE_PATH} +) + +set(standby_read_DEF_OPTIONS ${MACRO_OPTIONS}) +set(standby_read_COMPILE_OPTIONS ${OPTIMIZE_OPTIONS} ${OS_OPTIONS} ${PROTECT_OPTIONS} ${WARNING_OPTIONS} ${BIN_SECURE_OPTIONS} ${CHECK_OPTIONS}) +set(standby_read_LINK_OPTIONS ${BIN_LINK_OPTIONS}) +add_static_objtarget(gausskernel_storage_access_redo_standby_read TGT_standby_read_SRC TGT_standby_read_INC "${standby_read_DEF_OPTIONS}" "${standby_read_COMPILE_OPTIONS}" "${standby_read_LINK_OPTIONS}") diff --git a/src/gausskernel/storage/access/redo/standby_read/Makefile b/src/gausskernel/storage/access/redo/standby_read/Makefile new file mode 100644 index 000000000..236784529 --- /dev/null +++ b/src/gausskernel/storage/access/redo/standby_read/Makefile @@ -0,0 +1,37 @@ +# +# Copyright (c) 2020 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# --------------------------------------------------------------------------------------- +# +# Makefile-- +# Makefile for access/psort +# +# IDENTIFICATION +# src/backend/access/psort/Makefile +# +#------------------------------------------------------------------------- + +subdir = src/gausskernel/storage/access/redo/standby_read +top_builddir = ../../../../../.. +include $(top_builddir)/src/Makefile.global + +ifneq "$(MAKECMDGOALS)" "clean" + ifneq "$(MAKECMDGOALS)" "distclean" + ifneq "$(shell which g++ |grep hutaf_llt |wc -l)" "1" + -include $(DEPEND) + endif + endif +endif +OBJS = base_page_proc.o block_info_proc.o lsn_info_double_list.o lsn_info_proc.o standby_read_interface.o + +include $(top_srcdir)/src/gausskernel/common.mk diff --git a/src/gausskernel/storage/access/redo/standby_read/base_page_proc.cpp b/src/gausskernel/storage/access/redo/standby_read/base_page_proc.cpp new file mode 100644 index 000000000..514457106 --- /dev/null +++ b/src/gausskernel/storage/access/redo/standby_read/base_page_proc.cpp @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * base_page_proc.cpp + * + * IDENTIFICATION + * src/gausskernel/storage/access/redo/standby_read/base_page_proc.cpp + * + * ------------------------------------------------------------------------- + */ + +#include "postgres.h" +#include "access/extreme_rto/batch_redo.h" +#include "access/extreme_rto/dispatcher.h" +#include "access/extreme_rto/page_redo.h" +#include "access/extreme_rto/standby_read/standby_read_base.h" +#include "storage/buf/buf_internals.h" + +namespace extreme_rto_standby_read { + +inline RelFileNode make_base_page_relfilenode(uint32 batch_id, uint32 redo_worker_id, BasePagePosition position) +{ + RelFileNode rnode; + rnode.spcNode = EXRTO_BASE_PAGE_SPACE_OID; + rnode.dbNode = (batch_id << LOW_WORKERID_BITS) | redo_worker_id; + rnode.relNode = (uint32)((position / BLCKSZ) >> UINT64_HALF); + rnode.bucketNode = InvalidBktId; + rnode.opt = DefaultFileNodeOpt; + + return rnode; +} + +Buffer buffer_read_base_page(uint32 batch_id, uint32 redo_id, BasePagePosition position, ReadBufferMode mode) +{ + RelFileNode rnode = make_base_page_relfilenode(batch_id, redo_id, position); + BlockNumber blocknum = (BlockNumber)(position / BLCKSZ); + bool hit = false; + SMgrRelation smgr = smgropen(rnode, InvalidBackendId); + Buffer buffer = + ReadBuffer_common(smgr, RELPERSISTENCE_PERMANENT, MAIN_FORKNUM, blocknum, mode, NULL, &hit, NULL); + if (buffer == InvalidBuffer) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), + (errmsg("invalid buffer when read base page, batch_id: %u, redo_worker_id: %u, blocknum: %lu", + batch_id, redo_id, position / BLCKSZ)))); + } + + return buffer; +} + +void generate_base_page(StandbyReadMetaInfo* meta_info, const Page src_page) +{ + BasePagePosition position = meta_info->base_page_next_position; + + Buffer dest_buf = buffer_read_base_page(meta_info->batch_id, meta_info->redo_id, position, RBM_ZERO_AND_LOCK); + + Page dest_page = BufferGetPage(dest_buf); + errno_t rc = memcpy_s(dest_page, BLCKSZ, src_page, BLCKSZ); + securec_check(rc, "\0", "\0"); + MarkBufferDirty(dest_buf); + UnlockReleaseBuffer(dest_buf); + + meta_info->base_page_next_position += BLCKSZ; +} + +void read_base_page(const BufferTag& buf_tag, BasePagePosition position, BufferDesc* dest_buf_desc) +{ + extreme_rto::RedoItemTag redo_item_tag; + const uint32 worker_num_per_mng = extreme_rto::get_page_redo_worker_num_per_manager(); + + /* batch id and worker id start from 1 when reading a page */ + uint32 batch_id = extreme_rto::GetSlotId(buf_tag.rnode, 0, 0, extreme_rto::GetBatchCount()) + 1; + INIT_REDO_ITEM_TAG(redo_item_tag, buf_tag.rnode, buf_tag.forkNum, buf_tag.blockNum); + uint32 redo_worker_id = extreme_rto::GetWorkerId(&redo_item_tag, worker_num_per_mng) + 1; + + Buffer buffer = buffer_read_base_page(batch_id, redo_worker_id, position, RBM_NORMAL); + + LockBuffer(buffer, BUFFER_LOCK_SHARE); + Page src_page = BufferGetPage(buffer); + Size page_size = BufferGetPageSize(buffer); + Page dest_page = (Page)BufHdrGetBlock(dest_buf_desc); + errno_t rc = memcpy_s(dest_page, page_size, src_page, page_size); + securec_check(rc, "\0", "\0"); + UnlockReleaseBuffer(buffer); +} + +void recycle_base_page_file(uint32 batch_id, uint32 redo_id, BasePagePosition recycle_pos) +{ + RelFileNode rnode = make_base_page_relfilenode(batch_id, redo_id, recycle_pos); + SMgrRelation smgr = smgropen(rnode, InvalidBackendId); + + smgrdounlink(smgr, true, (BlockNumber)(recycle_pos / BLCKSZ)); +} + +} // namespace extreme_rto_standby_read + diff --git a/src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp b/src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp new file mode 100644 index 000000000..356a6687b --- /dev/null +++ b/src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp @@ -0,0 +1,299 @@ +/* + * Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * block_info_proc.cpp + * + * IDENTIFICATION + * src/gausskernel/storage/recovery/parallel/blocklevel/standby_read/block_info_proc.cpp + * + * ------------------------------------------------------------------------- + */ + +#include +#include "access/extreme_rto/standby_read/block_info_meta.h" +#include "access/extreme_rto/standby_read/lsn_info_meta.h" +#include "storage/smgr/relfilenode.h" + +namespace extreme_rto_standby_read { + +void block_info_page_init(Page page) +{ + static_assert(sizeof(BlockInfoPageHeader) == BLOCK_INFO_HEAD_SIZE, "BlockInfoPageHeader size is not 64 bytes"); + static_assert(sizeof(BlockMetaInfo) == BLOCK_INFO_SIZE, "BlockMetaInfo size is not 64 bytes"); + + BlockInfoPageHeader* page_header = (BlockInfoPageHeader*)page; + errno_t ret = memset_s(page_header, BLCKSZ, 0, BLCKSZ); + securec_check(ret, "", ""); + page_header->flags |= BLOCK_INFO_PAGE_VALID_FLAG; + page_header->version = BLOCK_INFO_PAGE_VERSION; +} + +inline BlockNumber data_block_number_to_meta_page_number(BlockNumber block_num) +{ + return block_num / BLOCK_INFO_NUM_PER_PAGE; +} + +inline uint32 block_info_meta_page_offset(BlockNumber block_num) +{ + return (block_num % BLOCK_INFO_NUM_PER_PAGE) * BLOCK_INFO_SIZE + BLOCK_INFO_HEAD_SIZE; +} + +// get page, just have pin, no lock +BlockMetaInfo* get_block_meta_info_by_relfilenode( + const BufferTag& buf_tag, BufferAccessStrategy strategy, ReadBufferMode mode, Buffer* buffer) +{ + RelFileNode standby_read_rnode = buf_tag.rnode; + standby_read_rnode.spcNode = EXRTO_BLOCK_INFO_SPACE_OID; + SMgrRelation smgr = smgropen(standby_read_rnode, InvalidBackendId); + bool hit = false; + + BlockNumber meta_block_num = data_block_number_to_meta_page_number(buf_tag.blockNum); + *buffer = ReadBuffer_common(smgr, 0, buf_tag.forkNum, meta_block_num, mode, strategy, &hit, NULL); + + if (*buffer == InvalidBuffer) { + return NULL; + } + + Page page = BufferGetPage(*buffer); + if (!is_block_info_page_valid((BlockInfoPageHeader*)page)) { + if (mode == RBM_NORMAL) { + ReleaseBuffer(*buffer); + return NULL; + } + } + + uint32 offset = block_info_meta_page_offset(buf_tag.blockNum); + BlockMetaInfo *block_info = ((BlockMetaInfo*)(page + offset)); + if (!is_block_meta_info_valid(block_info) && mode == RBM_NORMAL) { + ReleaseBuffer(*buffer); + + return NULL; + } + + return block_info; +} + +void init_block_info(BlockMetaInfo* block_info, XLogRecPtr max_lsn) +{ + errno_t ret = memset_s(block_info, BLOCK_INFO_SIZE, 0, BLOCK_INFO_SIZE); + securec_check(ret, "", ""); + block_info->timeline = t_thrd.shemem_ptr_cxt.ControlFile->timeline; + block_info->flags |= BLOCK_INFO_NODE_VALID_FLAG; + lsn_info_list_init(&block_info->lsn_info_list); + lsn_info_list_init(&block_info->base_page_info_list); + block_info->max_lsn = max_lsn; // just for update first base page info' lsn + block_info->min_lsn = max_lsn; +} + +void insert_lsn_to_block_info( + StandbyReadMetaInfo* meta_info, const BufferTag& buf_tag, const Page base_page, XLogRecPtr next_lsn) +{ + Buffer block_info_buf = InvalidBuffer; + BlockMetaInfo* block_info = get_block_meta_info_by_relfilenode(buf_tag, NULL, RBM_ZERO_ON_ERROR, &block_info_buf); + if (unlikely(block_info == NULL || block_info_buf == InvalidBuffer)) { + ereport(PANIC, (errmsg("insert lsn failed,block invalid %u/%u/%u %d %u", buf_tag.rnode.spcNode, + buf_tag.rnode.dbNode, buf_tag.rnode.relNode, buf_tag.forkNum, buf_tag.blockNum))); + } + LockBuffer(block_info_buf, BUFFER_LOCK_EXCLUSIVE); + Page page = BufferGetPage(block_info_buf); + XLogRecPtr current_page_lsn = PageGetLSN(base_page); + if (!is_block_meta_info_valid(block_info)) { + if (!is_block_info_page_valid((BlockInfoPageHeader*)page)) { + block_info_page_init(page); + } + + init_block_info(block_info, current_page_lsn); + } + + if (block_info->record_num == 0 || + (block_info->record_num % (uint32)g_instance.attr.attr_storage.base_page_saved_interval) == 0) { + insert_base_page_to_lsn_info(meta_info, &block_info->lsn_info_list, &block_info->base_page_info_list, buf_tag, + base_page, current_page_lsn, next_lsn); + } else { + insert_lsn_to_lsn_info(meta_info, &block_info->lsn_info_list, next_lsn); + } + + Assert(block_info->max_lsn <= next_lsn); + block_info->max_lsn = next_lsn; + + ++(block_info->record_num); + + standby_read_meta_page_set_lsn(page, next_lsn); + MarkBufferDirty(block_info_buf); + UnlockReleaseBuffer(block_info_buf); +} + +StandbyReadRecyleState recyle_block_info( + const BufferTag& buf_tag, LsnInfoPosition base_page_info_pos, XLogRecPtr next_base_page_lsn, XLogRecPtr recyle_lsn) +{ + Buffer buffer = InvalidBuffer; + BlockMetaInfo* block_meta_info = get_block_meta_info_by_relfilenode(buf_tag, NULL, RBM_NORMAL, &buffer); + if ((block_meta_info == NULL) || (buffer == InvalidBuffer)) { + // no block info, should not at this branch + ereport(WARNING, (errmsg("block meta is invalid %u/%u/%u %d %u", buf_tag.rnode.spcNode, buf_tag.rnode.dbNode, + buf_tag.rnode.relNode, buf_tag.forkNum, buf_tag.blockNum))); + return STANDBY_READ_RECLYE_ALL; + } + StandbyReadRecyleState stat = STANDBY_READ_RECLYE_NONE; + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + Assert(((block_meta_info->flags & BLOCK_INFO_NODE_VALID_FLAG) == BLOCK_INFO_NODE_VALID_FLAG)); + if (XLByteLT(block_meta_info->max_lsn, recyle_lsn)) { + block_meta_info->flags &= ~BLOCK_INFO_NODE_VALID_FLAG; + stat = STANDBY_READ_RECLYE_ALL; + MarkBufferDirty(buffer); + } else if (XLogRecPtrIsValid(next_base_page_lsn)) { + LsnInfoPosition min_page_info_pos = LSN_INFO_LIST_HEAD; + XLogRecPtr min_lsn = InvalidXLogRecPtr; + recycle_one_lsn_info_list(buf_tag, base_page_info_pos, recyle_lsn, &min_page_info_pos, &min_lsn); + + Assert(INFO_POSITION_IS_VALID(min_page_info_pos)); + if (block_meta_info->base_page_info_list.next != min_page_info_pos) { + block_meta_info->min_lsn = min_lsn; + block_meta_info->lsn_info_list.next = min_page_info_pos; + block_meta_info->base_page_info_list.next = min_page_info_pos; + stat = STANDBY_READ_RECLYE_UPDATE; + MarkBufferDirty(buffer); + } + } + UnlockReleaseBuffer(buffer); + return stat; +} + +static void reset_tmp_lsn_info_array(StandbyReadLsnInfoArray* lsn_info) +{ + lsn_info->lsn_num = 0; + lsn_info->base_page_lsn = InvalidXLogRecPtr; + if (lsn_info->lsn_array == NULL) { + uint32 max_save_nums = (uint32)g_instance.attr.attr_storage.base_page_saved_interval; + lsn_info->lsn_array = (XLogRecPtr*)MemoryContextAlloc( + THREAD_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_STORAGE), sizeof(XLogRecPtr) * max_save_nums); + } +} + +bool get_page_lsn_info(const BufferTag& buf_tag, BufferAccessStrategy strategy, XLogRecPtr read_lsn, + StandbyReadLsnInfoArray* lsn_info) +{ + Buffer buf; + BlockMetaInfo* block_meta_info = get_block_meta_info_by_relfilenode(buf_tag, strategy, RBM_NORMAL, &buf); + if (block_meta_info == NULL) { + return false; + } + + LockBuffer(buf, BUFFER_LOCK_SHARE); + + if (XLByteLT(read_lsn, block_meta_info->min_lsn)) { + UnlockReleaseBuffer(buf); + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), + (errmsg("block old version can not found %u/%u/%u %d %u read lsn %lu, min lsn %lu", + buf_tag.rnode.spcNode, buf_tag.rnode.dbNode, buf_tag.rnode.relNode, buf_tag.forkNum, + buf_tag.blockNum, read_lsn, block_meta_info->min_lsn)))); + return false; + } + + Assert(block_meta_info->base_page_info_list.prev != LSN_INFO_LIST_HEAD); + reset_tmp_lsn_info_array(lsn_info); + get_lsn_info_for_read(buf_tag, block_meta_info->base_page_info_list.prev, lsn_info, read_lsn); + UnlockReleaseBuffer(buf); + return true; +} + +/* + * recycle one block info file + * rnode: database oid. + */ +void remove_one_block_info_file(const RelFileNode rnode) +{ + DropRelFileNodeShareBuffers(rnode, MAIN_FORKNUM, 0); + DropRelFileNodeShareBuffers(rnode, FSM_FORKNUM, 0); + DropRelFileNodeShareBuffers(rnode, VISIBILITYMAP_FORKNUM, 0); + + SMgrRelation srel = smgropen(rnode, InvalidBackendId); + smgrdounlink(srel, true); + smgrclose(srel); +} +/* + * recycle all relation files when drop db occurs. + * db_id: database oid. + */ +void remove_block_meta_info_files_of_db(Oid db_oid, Oid rel_oid) +{ + char pathbuf[EXRTO_FILE_PATH_LEN]; + char **filenames; + char **filename; + struct stat statbuf; + /* get block info file directory */ + char exrto_block_info_dir[EXRTO_FILE_PATH_LEN] = {0}; + int rc = snprintf_s(exrto_block_info_dir, EXRTO_FILE_PATH_LEN, EXRTO_FILE_PATH_LEN - 1, "%s/%s", EXRTO_FILE_DIR, + EXRTO_FILE_SUB_DIR[BLOCK_INFO_META]); + securec_check_ss(rc, "", ""); + /* get all files' name from block meta file directory */ + filenames = pgfnames(exrto_block_info_dir); + if (filenames == NULL) { + return; + } + char target_prefix[EXRTO_FILE_PATH_LEN] = {0}; + if (rel_oid != InvalidOid) { + rc = sprintf_s(target_prefix, EXRTO_FILE_PATH_LEN, "%u_%u_", db_oid, rel_oid); + } else { + rc = sprintf_s(target_prefix, EXRTO_FILE_PATH_LEN, "%u_", db_oid); + } + securec_check_ss(rc, "", ""); + /* use the prefix name to match up files we want to delete */ + size_t prefix_len = strlen(target_prefix); + for (filename = filenames; *filename != NULL; filename++) { + char *fname = *filename; + size_t fname_len = strlen(fname); + /* + * the length of prefix is less than the length of file name and must be the same under the same prefix_len + */ + if (prefix_len >= fname_len || strncmp(target_prefix, fname, prefix_len) != 0) { + continue; + } + rc = + snprintf_s(pathbuf, EXRTO_FILE_PATH_LEN, EXRTO_FILE_PATH_LEN - 1, "%s/%s", exrto_block_info_dir, *filename); + securec_check_ss(rc, "", ""); + /* may be can be some error */ + if (lstat(pathbuf, &statbuf) != 0) { + if (errno != ENOENT) { +#ifndef FRONTEND + ereport(WARNING, (errmsg("could not stat file or directory \"%s\" \n", pathbuf))); +#else + fprintf(stderr, _("could not stat file or directory \"%s\": %s\n"), pathbuf, gs_strerror(errno)); +#endif + } + continue; + } + /* if the file is a directory, don't touch it */ + if (S_ISDIR(statbuf.st_mode)) { + /* skip dir */ + continue; + } + /* delete this file we found */ + if (unlink(pathbuf) != 0) { + if (errno != ENOENT) { +#ifndef FRONTEND + ereport(WARNING, (errmsg("could not remove file or directory \"%s\" ", pathbuf))); +#else + fprintf(stderr, _("could not remove file or directory \"%s\": %s\n"), pathbuf, gs_strerror(errno)); +#endif + } + } + } + pgfnames_cleanup(filenames); + return; +} + +} // namespace extreme_rto_standby_read + diff --git a/src/gausskernel/storage/access/redo/standby_read/lsn_info_double_list.cpp b/src/gausskernel/storage/access/redo/standby_read/lsn_info_double_list.cpp new file mode 100644 index 000000000..bffb2be17 --- /dev/null +++ b/src/gausskernel/storage/access/redo/standby_read/lsn_info_double_list.cpp @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * lsn_info_double_list.cpp + * + * IDENTIFICATION + * src/gausskernel/storage/access/redo/standby_read/lsn_info_double_list.cpp + * + * ------------------------------------------------------------------------- + */ + +#include "access/extreme_rto/standby_read/lsn_info_double_list.h" +#include "access/extreme_rto/standby_read/lsn_info_meta.h" + +namespace extreme_rto_standby_read { + +void lsn_info_list_init(LsnInfoDoubleList* node) +{ + node->next = LSN_INFO_LIST_HEAD; + node->prev = LSN_INFO_LIST_HEAD; +} + +/* + * modify the tail of list to link new node (block meta table's page lock is held) + */ +void info_list_modify_old_tail(StandbyReadMetaInfo *meta_info, LsnInfoPosition old_tail_pos, + LsnInfoPosition insert_pos, XLogRecPtr current_page_lsn, XLogRecPtr next_lsn, bool is_lsn_info) +{ + Page page = NULL; + LsnInfo lsn_info = NULL; + BasePageInfo base_page_info = NULL; + uint32 batch_id = meta_info->batch_id; + uint32 worker_id = meta_info->redo_id; + Buffer buffer = InvalidBuffer; + uint32 offset; + + page = get_lsn_info_page(batch_id, worker_id, old_tail_pos, RBM_ZERO_ON_ERROR, &buffer); + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + + offset = lsn_info_postion_to_offset(old_tail_pos); + Assert(offset >= LSN_INFO_HEAD_SIZE); + Assert(offset % LSN_INFO_NODE_SIZE == 0); + if (is_lsn_info) { + lsn_info = (LsnInfo)(page + offset); + Assert(lsn_info->lsn_list.next == LSN_INFO_LIST_HEAD); + lsn_info->lsn_list.next = insert_pos; + Assert(is_lsn_info_node_valid(lsn_info->flags)); + } else { + base_page_info = (BasePageInfo)(page + offset); + Assert(base_page_info->base_page_list.next == LSN_INFO_LIST_HEAD); + base_page_info->base_page_list.next = insert_pos; + base_page_info->next_base_page_lsn = current_page_lsn; + Assert(is_lsn_info_node_valid(base_page_info->lsn_info_node.flags)); + Assert(XLByteLT(base_page_info->cur_page_lsn, current_page_lsn)); + } + + standby_read_meta_page_set_lsn(page, next_lsn); + MarkBufferDirty(buffer); + UnlockReleaseBuffer(buffer); +} + +} // namespace extreme_rto_standby_read diff --git a/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp b/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp new file mode 100644 index 000000000..765538955 --- /dev/null +++ b/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp @@ -0,0 +1,650 @@ +/* + * Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * lsn_info_proc.cpp + * + * IDENTIFICATION + * src/gausskernel/storage/recovery/parallel/blocklevel/standby_read/lsn_info_proc.cpp + * + * ------------------------------------------------------------------------- + */ + +#include "postgres.h" +#include "access/extreme_rto/batch_redo.h" +#include "access/extreme_rto/dispatcher.h" +#include "access/extreme_rto/page_redo.h" +#include "access/extreme_rto/standby_read/block_info_meta.h" +#include "access/extreme_rto/standby_read/lsn_info_double_list.h" +#include "access/extreme_rto/standby_read/lsn_info_meta.h" +#include "access/extreme_rto/standby_read/standby_read_base.h" + +namespace extreme_rto_standby_read { + +void lsn_info_page_init(Page page) +{ + static_assert(sizeof(LsnInfoPageHeader) == LSN_INFO_HEAD_SIZE, "LsnInfoPageHeader size is not 64 bytes"); + static_assert(sizeof(LsnInfoNode) == LSN_INFO_NODE_SIZE, "LsnInfoNode size is not 64 bytes"); + static_assert(sizeof(BasePageInfoNode) == BASE_PAGE_INFO_NODE_SIZE, "BasePageInfoNode size is not 128 bytes"); + + LsnInfoPageHeader *page_header = (LsnInfoPageHeader *)page; + errno_t ret = memset_s(page_header, BLCKSZ, 0, BLCKSZ); + securec_check(ret, "", ""); + page_header->flags |= LSN_INFO_PAGE_VALID_FLAG; + page_header->version = LSN_INFO_PAGE_VERSION; +} + +void lsn_info_init(LsnInfo lsn_info) +{ + errno_t ret = memset_s(lsn_info, LSN_INFO_NODE_SIZE, 0, LSN_INFO_NODE_SIZE); + securec_check(ret, "", ""); + + lsn_info->flags |= LSN_INFO_NODE_VALID_FLAG; + lsn_info->type = LSN_INFO_TYPE_LSNS; + lsn_info_list_init(&lsn_info->lsn_list); +} +void base_page_info_init(BasePageInfo base_page_info) +{ + errno_t ret = memset_s(base_page_info, BASE_PAGE_INFO_NODE_SIZE, 0, BASE_PAGE_INFO_NODE_SIZE); + securec_check(ret, "", ""); + + base_page_info->lsn_info_node.flags |= LSN_INFO_NODE_VALID_FLAG; + base_page_info->lsn_info_node.type = LSN_INFO_TYPE_BASE_PAGE; + lsn_info_list_init(&base_page_info->lsn_info_node.lsn_list); + lsn_info_list_init(&base_page_info->base_page_list); +} + +RelFileNode make_lsn_info_relfilenode(uint32 batch_id, uint32 worker_id, LsnInfoPosition position) +{ + RelFileNode rnode = {0}; + rnode.spcNode = EXRTO_LSN_INFO_SPACE_OID; + rnode.dbNode = (batch_id << LOW_WORKERID_BITS) | worker_id; + rnode.relNode = (uint32)((position / BLCKSZ) >> UINT64_HALF); + rnode.bucketNode = InvalidBktId; + rnode.opt = DefaultFileNodeOpt; + + return rnode; +} + +Page get_lsn_info_page(uint32 batch_id, uint32 worker_id, LsnInfoPosition position, ReadBufferMode mode, + Buffer* buffer) +{ + RelFileNode rnode; + BlockNumber block_num; + bool hit = false; + Page page = NULL; + + rnode = make_lsn_info_relfilenode(batch_id, worker_id, position); + block_num = (uint32)(position / BLCKSZ); /* high 32 bits are stored in the relNode. */ + + SMgrRelation smgr = smgropen(rnode, InvalidBackendId); + *buffer = ReadBuffer_common(smgr, RELPERSISTENCE_PERMANENT, MAIN_FORKNUM, block_num, mode, NULL, &hit, NULL); + + if (*buffer == InvalidBuffer) { + ereport(WARNING, (errcode_for_file_access(), + errmsg("block is invalid %u/%u/%u %d %u, batch_id: %u, redo_worker_id: %u", + rnode.spcNode, rnode.dbNode, rnode.relNode, MAIN_FORKNUM, block_num, + batch_id, worker_id))); + return NULL; + } + + page = BufferGetPage(*buffer); + if (!is_lsn_info_page_valid((LsnInfoPageHeader*)page)) { + if (mode == RBM_NORMAL) { + ReleaseBuffer(*buffer); + *buffer = InvalidBuffer; + return NULL; + } + /* make sure to make buffer dirty outside */ + lsn_info_page_init(page); + } + + return page; +} + +LsnInfoPosition create_lsn_info_node(StandbyReadMetaInfo *meta_info, LsnInfoPosition old_tail_pos, + XLogRecPtr next_lsn, bool create_in_old_page, Page old_page) +{ + Page page = NULL; + LsnInfo lsn_info = NULL; + uint32 batch_id = meta_info->batch_id; + uint32 worker_id = meta_info->redo_id; + LsnInfoPosition insert_pos = meta_info->lsn_table_next_position; + Buffer buffer = InvalidBuffer; + uint32 offset; + + offset = lsn_info_postion_to_offset(insert_pos); + if (offset == 0) { + insert_pos += LSN_INFO_HEAD_SIZE; /* actual insert position */ + offset += LSN_INFO_HEAD_SIZE; + } + Assert(offset % LSN_INFO_NODE_SIZE == 0); + + if (create_in_old_page) { + /* in old page, buffer is already locked */ + lsn_info = (LsnInfo)(old_page + offset); + } else { + page = get_lsn_info_page(batch_id, worker_id, insert_pos, RBM_ZERO_ON_ERROR, &buffer); + + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + lsn_info = (LsnInfo)(page + offset); + } + + lsn_info_init(lsn_info); + lsn_info->lsn[lsn_info->used] = next_lsn; + lsn_info->used++; + lsn_info->lsn_list.prev = old_tail_pos; + + if (!create_in_old_page) { + standby_read_meta_page_set_lsn(page, next_lsn); + MarkBufferDirty(buffer); + UnlockReleaseBuffer(buffer); + } + /* update meta info */ + meta_info->lsn_table_next_position = insert_pos + LSN_INFO_NODE_SIZE; + + return insert_pos; +} + +void insert_lsn_to_lsn_info(StandbyReadMetaInfo *meta_info, LsnInfoDoubleList *lsn_head, XLogRecPtr next_lsn) +{ + Page page = NULL; + LsnInfo lsn_info = NULL; + uint32 batch_id = meta_info->batch_id; + uint32 worker_id = meta_info->redo_id; + LsnInfoPosition tail_pos = lsn_head->prev; /* lsn info node tail */ + LsnInfoPosition insert_pos = meta_info->lsn_table_next_position; + Buffer buffer = InvalidBuffer; + uint32 offset; + + Assert(!INFO_POSITION_IS_INVALID(tail_pos)); + page = get_lsn_info_page(batch_id, worker_id, tail_pos, RBM_ZERO_ON_ERROR, &buffer); + + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + offset = lsn_info_postion_to_offset(tail_pos); + lsn_info = (LsnInfo)(page + offset); + Assert(offset >= LSN_INFO_HEAD_SIZE); + Assert(offset % LSN_INFO_NODE_SIZE == 0); + Assert(is_lsn_info_node_valid(lsn_info->flags)); + Assert(lsn_info->lsn_list.next == LSN_INFO_LIST_HEAD); + if (lsn_info->used < LSN_NUM_PER_NODE) { + lsn_info->lsn[lsn_info->used] = next_lsn; + lsn_info->used++; + + standby_read_meta_page_set_lsn(page, next_lsn); + MarkBufferDirty(buffer); + UnlockReleaseBuffer(buffer); + } else { + /* + * There is no free space in the old lsn info node, create a new one. + */ + bool create_in_old_page = (insert_pos / BLCKSZ) == (tail_pos / BLCKSZ); + /* insert position maybe changed */ + insert_pos = create_lsn_info_node(meta_info, tail_pos, next_lsn, create_in_old_page, page); + + /* modify lsn info list */ + lsn_info->lsn_list.next = insert_pos; + standby_read_meta_page_set_lsn(page, next_lsn); + MarkBufferDirty(buffer); + UnlockReleaseBuffer(buffer); + /* update lsn info tail in block info meta */ + lsn_head->prev = insert_pos; + } +} + +LsnInfoPosition create_base_page_info_node(StandbyReadMetaInfo *meta_info, + LsnInfoPosition old_lsn_tail_pos, LsnInfoPosition old_base_page_tail_pos, const BufferTag* buf_tag, + XLogRecPtr current_page_lsn, XLogRecPtr next_lsn) +{ + Page page = NULL; + BasePageInfo base_page_info = NULL; + uint32 batch_id = meta_info->batch_id; + uint32 worker_id = meta_info->redo_id; + LsnInfoPosition insert_pos = meta_info->lsn_table_next_position; + BasePagePosition base_page_pos = meta_info->base_page_next_position; + Buffer buffer = InvalidBuffer; + uint32 offset; + uint32 remain_size; + + /* + * If there is not enough space in current page, we insert base page info node in next page. + */ + remain_size = BLCKSZ - insert_pos % BLCKSZ; + if (remain_size < BASE_PAGE_INFO_NODE_SIZE) { + Assert(remain_size == LSN_INFO_NODE_SIZE); + insert_pos += LSN_INFO_NODE_SIZE; /* switch to next page */ + } + + offset = lsn_info_postion_to_offset(insert_pos); + Assert(offset % LSN_INFO_NODE_SIZE == 0); + if (offset == 0) { + insert_pos += LSN_INFO_HEAD_SIZE; /* actual insert position */ + offset += LSN_INFO_HEAD_SIZE; + } + + page = get_lsn_info_page(batch_id, worker_id, insert_pos, RBM_ZERO_ON_ERROR, &buffer); + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + + base_page_info = (BasePageInfo)(page + offset); + + base_page_info_init(base_page_info); + base_page_info->lsn_info_node.lsn_list.prev = old_lsn_tail_pos; + base_page_info->lsn_info_node.lsn[0] = next_lsn; + base_page_info->lsn_info_node.used++; + base_page_info->base_page_list.prev = old_base_page_tail_pos; + base_page_info->cur_page_lsn = current_page_lsn; + base_page_info->relfilenode = buf_tag->rnode; + base_page_info->fork_num = buf_tag->forkNum; + base_page_info->block_num = buf_tag->blockNum; + base_page_info->next_base_page_lsn = InvalidXLogRecPtr; + base_page_info->base_page_position = base_page_pos; + + set_base_page_map_bit(page, offset); + + standby_read_meta_page_set_lsn(page, next_lsn); + MarkBufferDirty(buffer); + UnlockReleaseBuffer(buffer); + + /* update meta info */ + meta_info->lsn_table_next_position = insert_pos + BASE_PAGE_INFO_NODE_SIZE; + + return insert_pos; +} + +void insert_base_page_to_lsn_info(StandbyReadMetaInfo *meta_info, LsnInfoDoubleList *lsn_head, + LsnInfoDoubleList *base_page_head, const BufferTag& buf_tag, const Page base_page, XLogRecPtr current_page_lsn, + XLogRecPtr next_lsn) +{ + LsnInfoPosition old_lsn_tail_pos = lsn_head->prev; + LsnInfoPosition old_base_page_tail_pos = base_page_head->prev; + LsnInfoPosition insert_pos; + + /* possibly modified meta_info */ + insert_pos = create_base_page_info_node(meta_info, old_lsn_tail_pos, old_base_page_tail_pos, &buf_tag, + current_page_lsn, next_lsn); + + /* modify old tail information of lsn info node and base page info node */ + if (old_lsn_tail_pos != LSN_INFO_LIST_HEAD) { + info_list_modify_old_tail(meta_info, old_lsn_tail_pos, insert_pos, current_page_lsn, next_lsn, true); + } + if (old_base_page_tail_pos != LSN_INFO_LIST_HEAD) { + info_list_modify_old_tail(meta_info, old_base_page_tail_pos, insert_pos, current_page_lsn, next_lsn, false); + } + + /* modify block info meta */ + lsn_head->prev = insert_pos; + base_page_head->prev = insert_pos; + + if (INFO_POSITION_IS_INVALID(lsn_head->next)) { + lsn_head->next = insert_pos; + } + if (INFO_POSITION_IS_INVALID(base_page_head->next)) { + base_page_head->next = insert_pos; + } + + /* generate base page */ + generate_base_page(meta_info, base_page); +} + +void get_lsn_info_for_read(const BufferTag& buf_tag, LsnInfoPosition latest_lsn_base_page_pos, + StandbyReadLsnInfoArray* lsn_info_list, XLogRecPtr read_lsn) +{ + BasePageInfo base_page_info = NULL; + LsnInfoPosition next_lsn_info_pos; + Buffer buffer; + + XLogRecPtr page_lsn; + XLogRecPtr xlog_lsn; + uint32 batch_id; + uint32 worker_id; + XLogRecPtr *lsn_arry = lsn_info_list->lsn_array; + + /* get batch id and page redo worker id */ + extreme_rto::RedoItemTag redo_item_tag; + const uint32 worker_num_per_mng = extreme_rto::get_page_redo_worker_num_per_manager(); + INIT_REDO_ITEM_TAG(redo_item_tag, buf_tag.rnode, buf_tag.forkNum, buf_tag.blockNum); + /* batch id and worker id start from 1 when reading a page */ + batch_id = extreme_rto::GetSlotId(buf_tag.rnode, 0, 0, extreme_rto::get_batch_redo_num()) + 1; + worker_id = extreme_rto::GetWorkerId(&redo_item_tag, worker_num_per_mng) + 1; + + /* find fisrt base page whose lsn less than read lsn form tail to head */ + do { + /* reach the end of the list */ + if (INFO_POSITION_IS_INVALID(latest_lsn_base_page_pos)) { + ereport(ERROR, ( + errmsg("can not find base page, block is %u/%u/%u %d %u, batch_id: %u, redo_worker_id: %u", + buf_tag.rnode.spcNode, buf_tag.rnode.dbNode, buf_tag.rnode.relNode, buf_tag.forkNum, + buf_tag.blockNum, batch_id, worker_id))); + break; + } + buffer = InvalidBuffer; + Page page = get_lsn_info_page(batch_id, worker_id, latest_lsn_base_page_pos, RBM_NORMAL, &buffer); + if (unlikely(page == NULL || buffer == InvalidBuffer)) { + ereport(ERROR, + (errmsg(EXRTOFORMAT("get_lsn_info_page failed, batch_id: %u, redo_id: %u, pos: %lu"), batch_id, + worker_id, latest_lsn_base_page_pos))); + } + LockBuffer(buffer, BUFFER_LOCK_SHARE); + + uint32 offset = lsn_info_postion_to_offset(latest_lsn_base_page_pos); + base_page_info = (BasePageInfo)(page + offset); + + page_lsn = base_page_info->cur_page_lsn; + lsn_info_list->base_page_pos = base_page_info->base_page_position; + lsn_info_list->base_page_lsn = base_page_info->cur_page_lsn; + Assert(is_base_page_type(base_page_info->lsn_info_node.type)); + + /* If we find the desired page, keep it locked */ + if (XLByteLT(page_lsn, read_lsn)) { + break; + } + UnlockReleaseBuffer(buffer); + latest_lsn_base_page_pos = base_page_info->base_page_list.prev; + } while (true); + + LsnInfo lsn_info = &base_page_info->lsn_info_node; + bool find_end = false; + uint32 lsn_num = 0; + do { + for (uint16 i = 0; i < lsn_info->used; ++i) { + xlog_lsn = lsn_info->lsn[i]; + if (XLByteLE(read_lsn, xlog_lsn)) { + find_end = true; + break; + } + + lsn_arry[lsn_num++] = xlog_lsn; + } + next_lsn_info_pos = lsn_info->lsn_list.next; + UnlockReleaseBuffer(buffer); + /* reach the end of the list */ + if (find_end || next_lsn_info_pos == LSN_INFO_LIST_HEAD) { + break; + } + + Page page = get_lsn_info_page(batch_id, worker_id, next_lsn_info_pos, RBM_NORMAL, &buffer); + if (unlikely(page == NULL || buffer == InvalidBuffer)) { + ereport(ERROR, + (errmsg(EXRTOFORMAT("get_lsn_info_page failed, batch_id: %u, redo_id: %u, pos: %lu"), batch_id, + worker_id, next_lsn_info_pos))); + } + Assert(buffer != InvalidBuffer); + LockBuffer(buffer, BUFFER_LOCK_SHARE); + + uint32 offset = lsn_info_postion_to_offset(next_lsn_info_pos); + lsn_info = (LsnInfo)(page + offset); + } while (true); + + lsn_info_list->lsn_num = lsn_num; +} + +static bool check_base_page_loc_valid(uint32 base_page_loc) +{ + if (base_page_loc < LSN_INFO_HEAD_SIZE || base_page_loc > BLCKSZ - BASE_PAGE_INFO_NODE_SIZE || + base_page_loc % LSN_INFO_HEAD_SIZE != 0) { + ereport(ERROR, (errmsg("invalid BasePageInfo location:%u, page size:%d", base_page_loc, BLCKSZ))); + return false; + } + return true; +} + +/* + * set LsnInfoPageHeader::base_page_map specific bit from 0 to 1. + * the bit is correspond to some 64bytes range space in this page. + * params explanation. + * page: some page block in RAM(one block occupies 8192bytes in memory). + * base_page_loc: the offset of some BasePageInfoNode object from the beginning of this page. + * LsnInfoPageHeader::base_page_map has 128 bit which is mapped to 8192bytes page. + * every bit represent 64 bytes (64 = 8192/128). + * we can assume bit 0 map to [0, 64) of the page; + * bit 1 map to [64, 128) of the page; + * ...... + * bit 127 map to [8128, 8192) of the page; + * LsnInfoPageHeader is the page header which occupies 64bytes, so bit 0 is always 0. + * LSN_INFO_HEAD_SIZE,LSN_INFO_NODE_SIZE,BASE_PAGE_INFO_NODE_SIZE must be integer mutiple of 64, + * so we can use base_page_map to map page memory. + */ +void set_base_page_map_bit(Page page, uint32 base_page_loc) +{ + /* + * make sure base_page_loc is in specific range + * base_page_loc must be an integer multiple of LSN_INFO_HEAD_SIZE + */ + check_base_page_loc_valid(base_page_loc); + + LsnInfoPageHeader *page_header = (LsnInfoPageHeader *)page; + uint8 *base_page_map = page_header->base_page_map; + uint32 which_bit = base_page_loc / LSN_INFO_NODE_SIZE; + uint32 which_bytes = which_bit / BYTE_BITS; // uint8 has 8 bits or 8*sizeof(uint8) bits + uint32 bit_offset = which_bit % BYTE_BITS; + base_page_map[which_bytes] |= ((uint8)((uint8)1 << bit_offset)); +} + +static void check_base_page_map_bit_loc_valid(uint32 which_bit) +{ + if (which_bit >= BASE_PAGE_MAP_SIZE * BYTE_BITS) { + ereport(ERROR, (errmsg("invalid base_page_map bit location:%u, " + "the valid range is [%u, %u).", which_bit, 0U, BASE_PAGE_MAP_SIZE * BYTE_BITS))); + } +} + +/* + * check if LsnInfoPageHeader::base_page_map specific bit equal to 1. + * page: the page in which LsnInfoPageHeader object you want to check. + * which_bit: the bit you want to check. + * if the target bit is equal to 1, return true. + */ +bool is_base_page_map_bit_set(Page page, uint32 which_bit) +{ + check_base_page_map_bit_loc_valid(which_bit); + + LsnInfoPageHeader *page_header = (LsnInfoPageHeader *)page; + uint8 *base_page_map = page_header->base_page_map; + uint32 which_bytes = which_bit / BYTE_BITS; // uint8 has 8 bits or 8*sizeof(uint8) bits + uint32 bit_offset = which_bit % BYTE_BITS; + return (base_page_map[which_bytes] & (((uint8)1) << bit_offset)) != 0; +} + +void recycle_lsn_info_file(uint32 batch_id, uint32 redo_id, BasePagePosition recycle_pos) +{ + RelFileNode rnode = make_lsn_info_relfilenode(batch_id, redo_id, recycle_pos); + SMgrRelation smgr = smgropen(rnode, InvalidBackendId); + + smgrdounlink(smgr, true, (BlockNumber)(recycle_pos / BLCKSZ)); +} + +void recycle_one_lsn_info_list(const BufferTag& buf_tag, LsnInfoPosition page_info_pos, + XLogRecPtr recycle_lsn, LsnInfoPosition *min_page_info_pos, XLogRecPtr *min_lsn) +{ + /* get batch id and page redo worker id */ + extreme_rto::RedoItemTag redo_item_tag; + const uint32 worker_num_per_mng = extreme_rto::get_page_redo_worker_num_per_manager(); + INIT_REDO_ITEM_TAG(redo_item_tag, buf_tag.rnode, buf_tag.forkNum, buf_tag.blockNum); + /* batch id and worker id start from 1 when reading a page */ + uint32 batch_id = extreme_rto::GetSlotId(buf_tag.rnode, 0, 0, extreme_rto::get_batch_redo_num()) + 1; + uint32 worker_id = extreme_rto::GetWorkerId(&redo_item_tag, worker_num_per_mng) + 1; + + while (INFO_POSITION_IS_VALID(page_info_pos)) { + Buffer buffer = InvalidBuffer; + Page page = get_lsn_info_page(batch_id, worker_id, page_info_pos, RBM_NORMAL, &buffer); + if (unlikely(page == NULL || buffer == InvalidBuffer)) { + ereport(PANIC, (errmsg(EXRTOFORMAT("get_lsn_info_page failed, batch_id: %u, redo_id: %u, pos: %lu"), + batch_id, worker_id, page_info_pos))); + } + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + + uint32 offset = lsn_info_postion_to_offset(page_info_pos); + BasePageInfo base_page_info = (BasePageInfo)(page + offset); + Assert(is_base_page_type(base_page_info->lsn_info_node.type)); + + *min_page_info_pos = page_info_pos; + *min_lsn = base_page_info->cur_page_lsn; + + /* retain a page version with page lsn less than recycle lsn */ + XLogRecPtr next_base_page_lsn = base_page_info->next_base_page_lsn; + if (XLogRecPtrIsInvalid(next_base_page_lsn) || XLByteLT(recycle_lsn, next_base_page_lsn)) { + UnlockReleaseBuffer(buffer); + break; + } + + base_page_info->lsn_info_node.flags &= ~LSN_INFO_NODE_VALID_FLAG; + page_info_pos = base_page_info->base_page_list.next; + MarkBufferDirty(buffer); + UnlockReleaseBuffer(buffer); + } +} + +void invalid_base_page_list(StandbyReadMetaInfo *meta_info, Buffer buffer, uint32 offset) +{ + LsnInfoPosition page_info_pos; + Page page = BufferGetPage(buffer); + BasePageInfo base_page_info = (BasePageInfo)(page + offset); + /* set invalid flags */ + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + base_page_info->lsn_info_node.flags &= ~LSN_INFO_NODE_VALID_FLAG; + page_info_pos = base_page_info->base_page_list.next; + MarkBufferDirty(buffer); + LockBuffer(buffer, BUFFER_LOCK_UNLOCK); /* keep buffer pinned */ + + uint32 batch_id = meta_info->batch_id; + uint32 worker_id = meta_info->redo_id; + while (INFO_POSITION_IS_VALID(page_info_pos)) { + page = get_lsn_info_page(batch_id, worker_id, page_info_pos, RBM_NORMAL, &buffer); + if (unlikely(page == NULL || buffer == InvalidBuffer)) { + ereport(PANIC, (errmsg(EXRTOFORMAT("get_lsn_info_page failed, batch_id: %u, redo_id: %u, pos: %lu"), + batch_id, worker_id, page_info_pos))); + } + offset = lsn_info_postion_to_offset(page_info_pos); + base_page_info = (BasePageInfo)(page + offset); + + /* unset valid flags */ + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + base_page_info->lsn_info_node.flags &= ~LSN_INFO_NODE_VALID_FLAG; + page_info_pos = base_page_info->base_page_list.next; + MarkBufferDirty(buffer); + UnlockReleaseBuffer(buffer); + } +} + +inline void update_recycle_lsn_per_worker(StandbyReadMetaInfo *meta_info, XLogRecPtr lsn) +{ + Assert(XLogRecPtrIsValid(lsn)); + if (XLogRecPtrIsInvalid(meta_info->recycle_lsn_per_worker) || + XLByteLT(meta_info->recycle_lsn_per_worker, lsn)) { + meta_info->recycle_lsn_per_worker = lsn; + } + ereport(LOG, (errmsg(EXRTOFORMAT( + "[exrto_recycle] update recycle lsn per worker , batch_id: %u, redo_id: %u, recycle lsn: %08X/%08X"), + meta_info->batch_id, meta_info->redo_id, (uint32)(lsn >> UINT64_HALF), (uint32)lsn))); +} + +bool recycle_one_lsn_info_page(StandbyReadMetaInfo *meta_info, XLogRecPtr recycle_lsn, + BasePagePosition *base_page_position) +{ + uint32 batch_id = meta_info->batch_id; + uint32 worker_id = meta_info->redo_id; + Buffer buffer = InvalidBuffer; + LsnInfoPosition recycle_pos = meta_info->lsn_table_recyle_position; + Page page = get_lsn_info_page(batch_id, worker_id, recycle_pos, RBM_NORMAL, &buffer); + if (unlikely(page == NULL || buffer == InvalidBuffer)) { + ereport(PANIC, (errmsg(EXRTOFORMAT("get_lsn_info_page failed, batch_id: %u, redo_id: %u, pos: %lu"), + batch_id, worker_id, recycle_pos))); + } + + bool buffer_is_locked = false; + /* skip page header */ + for (uint32 bit = 1; bit < BASE_PAGE_MAP_SIZE * BYTE_BITS; bit++) { + if (!buffer_is_locked) { + LockBuffer(buffer, BUFFER_LOCK_SHARE); + buffer_is_locked = true; + } + + if (!is_base_page_map_bit_set(page, bit)) { + continue; + } + uint32 offset = bit_to_offset(bit); + BasePageInfo base_page_info = (BasePageInfo)(page + offset); + LsnInfoPosition cur_base_page_info_pos = recycle_pos + offset; + Assert(is_base_page_type(base_page_info->lsn_info_node.type)); + + /* block meta file may be dropped */ + if (!is_lsn_info_node_valid(base_page_info->lsn_info_node.flags)) { + continue; + } + + /* retain a page version with page lsn less than recycle lsn */ + XLogRecPtr base_page_lsn = base_page_info->cur_page_lsn; + if (XLogRecPtrIsInvalid(base_page_lsn)) { + base_page_lsn = base_page_info->lsn_info_node.lsn[0]; + } + XLogRecPtr next_base_page_lsn = base_page_info->next_base_page_lsn; + *base_page_position = base_page_info->base_page_position; + if (XLogRecPtrIsValid(next_base_page_lsn) && XLByteLT(recycle_lsn, next_base_page_lsn)) { + update_recycle_lsn_per_worker(meta_info, base_page_lsn); + UnlockReleaseBuffer(buffer); + return false; + } + + BufferTag buf_tag; + INIT_BUFFERTAG(buf_tag, base_page_info->relfilenode, base_page_info->fork_num, base_page_info->block_num); + + LockBuffer(buffer, BUFFER_LOCK_UNLOCK); + buffer_is_locked = false; + + StandbyReadRecyleState stat = + recyle_block_info(buf_tag, cur_base_page_info_pos, next_base_page_lsn, recycle_lsn); + if (stat == STANDBY_READ_RECLYE_ALL) { + invalid_base_page_list(meta_info, buffer, offset); + } else if (stat == STANDBY_READ_RECLYE_NONE) { + Assert(XLogRecPtrIsInvalid(next_base_page_lsn)); + update_recycle_lsn_per_worker(meta_info, base_page_lsn); + ReleaseBuffer(buffer); + return false; + } + } + + if (buffer_is_locked) { + LockBuffer(buffer, BUFFER_LOCK_UNLOCK); + } + ReleaseBuffer(buffer); + return true; +} + +void standby_read_recyle_per_workers(StandbyReadMetaInfo *meta_info, XLogRecPtr recycle_lsn) +{ + Assert(meta_info->batch_id > 0); + Assert(meta_info->redo_id > 0); + bool recycle_next_page = true; + BasePagePosition base_page_position = meta_info->base_page_recyle_position; + + while (meta_info->lsn_table_recyle_position + BLCKSZ < meta_info->lsn_table_next_position) { + recycle_next_page = recycle_one_lsn_info_page(meta_info, recycle_lsn, &base_page_position); + if (!recycle_next_page) { + break; + } + /* update recycle position */ + meta_info->lsn_table_recyle_position += BLCKSZ; + Assert(meta_info->lsn_table_recyle_position % BLCKSZ == 0); + RedoInterruptCallBack(); + } + + meta_info->base_page_recyle_position = base_page_position; + Assert(meta_info->base_page_recyle_position % BLCKSZ == 0); + Assert(meta_info->base_page_recyle_position <= meta_info->base_page_next_position); + + recycle_lsn_info_file(meta_info->batch_id, meta_info->redo_id, meta_info->lsn_table_recyle_position); + recycle_base_page_file(meta_info->batch_id, meta_info->redo_id, meta_info->base_page_recyle_position); +} + +} // namespace extreme_rto_standby_read diff --git a/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp b/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp new file mode 100644 index 000000000..f64492a46 --- /dev/null +++ b/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp @@ -0,0 +1,351 @@ +/* + * Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * standby_read_interface.cpp + * + * IDENTIFICATION + * src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp + * + * ------------------------------------------------------------------------- + */ + +#include +#include "access/extreme_rto/page_redo.h" +#include "access/extreme_rto/standby_read/block_info_meta.h" +#include "access/extreme_rto/standby_read/lsn_info_meta.h" +#include "access/multi_redo_api.h" +#include "pgstat.h" +#include "storage/smgr/relfilenode.h" +#include "storage/buf/buf_internals.h" +#include "storage/buf/bufmgr.h" +#include "storage/smgr/segment.h" +#include "utils/rel.h" +#include "utils/palloc.h" +#include "access/extreme_rto/dispatcher.h" +#include "funcapi.h" + +const char* EXRTO_BASE_PAGE_SUB_DIR = "base_page"; +const char* EXRTO_LSN_INFO_SUB_DIR = "lsn_info_meta"; +const char* EXRTO_BLOCK_INFO_SUB_DIR = "block_info_meta"; +const char* EXRTO_FILE_SUB_DIR[] = { + EXRTO_BASE_PAGE_SUB_DIR, EXRTO_LSN_INFO_SUB_DIR, EXRTO_BLOCK_INFO_SUB_DIR}; +const uint32 EXRTO_FILE_PATH_LEN = 1024; + +void make_standby_read_node(XLogRecPtr read_lsn, RelFileNode& read_node) +{ + read_node.spcNode = (Oid)(read_lsn >> 32); + read_node.dbNode = (Oid)(read_lsn); + read_node.relNode = InvalidOid; // make sure it can be InvalidOid or not + read_node.opt = 0; + read_node.bucketNode = InvalidBktId; +} + +BufferDesc* alloc_standby_read_buf( + const BufferTag& buf_tag, BufferAccessStrategy strategy, bool& found, XLogRecPtr read_lsn) +{ + RelFileNode read_node; + make_standby_read_node(read_lsn, read_node); + BufferDesc* buf_desc = BufferAlloc(read_node, 0, buf_tag.forkNum, buf_tag.blockNum, strategy, &found, NULL); + + return buf_desc; +} + +Buffer get_newest_page_for_read(Relation reln, ForkNumber fork_num, BlockNumber block_num, ReadBufferMode mode, + BufferAccessStrategy strategy, XLogRecPtr read_lsn) +{ + bool hit = false; + + Buffer newest_buf = ReadBuffer_common( + reln->rd_smgr, reln->rd_rel->relpersistence, fork_num, block_num, mode, strategy, &hit, NULL); + if (BufferIsInvalid(newest_buf)) { + return InvalidBuffer; + } + + LockBuffer(newest_buf, BUFFER_LOCK_SHARE); + Page newest_page = BufferGetPage(newest_buf); + XLogRecPtr page_lsn = PageGetLSN(newest_page); + if (XLByteLT(read_lsn, page_lsn)) { + UnlockReleaseBuffer(newest_buf); + return InvalidBuffer; + } + + BufferTag buf_tag = { + .rnode = reln->rd_smgr->smgr_rnode.node, + .forkNum = fork_num, + .blockNum = block_num, + }; + ResourceOwnerEnlargeBuffers(t_thrd.utils_cxt.CurrentResourceOwner); + BufferDesc* buf_desc = alloc_standby_read_buf(buf_tag, strategy, hit, page_lsn); + + if (hit) { + UnlockReleaseBuffer(newest_buf); + return BufferDescriptorGetBuffer(buf_desc); + } + Page read_page = (Page)BufHdrGetBlock(buf_desc); + + errno_t rc = memcpy_s(read_page, BLCKSZ, newest_page, BLCKSZ); + securec_check(rc, "\0", "\0"); + UnlockReleaseBuffer(newest_buf); + buf_desc->extra->lsn_on_disk = PageGetLSN(read_page); +#ifdef USE_ASSERT_CHECKING + buf_desc->lsn_dirty = InvalidXLogRecPtr; +#endif + + TerminateBufferIO(buf_desc, false, (BM_VALID | BM_IS_TMP_BUF)); + return BufferDescriptorGetBuffer(buf_desc); +} + +Buffer standby_read_buf( + Relation reln, ForkNumber fork_num, BlockNumber block_num, ReadBufferMode mode, BufferAccessStrategy strategy) +{ + /* Open it at the smgr level */ + RelationOpenSmgr(reln); // need or not ????? + pgstat_count_buffer_read(reln); + pgstatCountBlocksFetched4SessionLevel(); + + if (RelationisEncryptEnable(reln)) { + reln->rd_smgr->encrypt = true; + } + + bool hit = false; + BufferTag buf_tag = { + .rnode = reln->rd_smgr->smgr_rnode.node, + .forkNum = fork_num, + .blockNum = block_num, + }; + XLogRecPtr read_lsn = t_thrd.proc->exrto_read_lsn; + if (read_lsn == InvalidXLogRecPtr) { + Assert(IsSystemRelation(reln)); + read_lsn = MAX_XLOG_REC_PTR; + } + + Buffer read_buf = get_newest_page_for_read(reln, fork_num, block_num, mode, strategy, read_lsn); + + if (read_buf != InvalidBuffer) { + // newest page's lsn smaller than read lsn + return read_buf; + } + ResourceOwnerEnlargeBuffers(t_thrd.utils_cxt.CurrentResourceOwner); + // read lsn info + StandbyReadLsnInfoArray *lsn_info = &t_thrd.exrto_recycle_cxt.lsn_info; + bool result = extreme_rto_standby_read::get_page_lsn_info(buf_tag, strategy, read_lsn, lsn_info); + if (!result) { + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + (errmsg("standby_read_buf couldnot found buf %u/%u/%u %d %u read lsn %lu", buf_tag.rnode.spcNode, + buf_tag.rnode.dbNode, buf_tag.rnode.relNode, buf_tag.forkNum, buf_tag.blockNum, read_lsn)))); + return InvalidBuffer; + } + + // read lsn info + XLogRecPtr expected_lsn = InvalidXLogRecPtr; + if (lsn_info->lsn_num == 0) { + expected_lsn = lsn_info->base_page_lsn; + } else { + Assert(lsn_info->lsn_array[lsn_info->lsn_num - 1] > 0); + Assert(lsn_info->lsn_array[lsn_info->lsn_num - 1] < read_lsn); + Assert(lsn_info->lsn_array[lsn_info->lsn_num - 1] >= lsn_info->base_page_lsn); + expected_lsn = lsn_info->lsn_array[lsn_info->lsn_num - 1]; + } + + BufferDesc* buf_desc = alloc_standby_read_buf(buf_tag, strategy, hit, expected_lsn); + + if (hit) { + return BufferDescriptorGetBuffer(buf_desc); + } + buffer_in_progress_pop(); + // read_base_page + extreme_rto_standby_read::read_base_page(buf_tag, lsn_info->base_page_pos, buf_desc); + if (lsn_info->lsn_num > 0) { + redo_target_page(buf_tag, lsn_info, BufferDescriptorGetBuffer(buf_desc)); + } + Page page = BufferGetPage(BufferDescriptorGetBuffer(buf_desc)); + buf_desc->extra->lsn_on_disk = PageGetLSN(page); +#ifdef USE_ASSERT_CHECKING + buf_desc->lsn_dirty = InvalidXLogRecPtr; +#endif + buffer_in_progress_push(); + TerminateBufferIO(buf_desc, false, (BM_VALID | BM_IS_TMP_BUF)); + + return BufferDescriptorGetBuffer(buf_desc); +} + +void make_exrto_file_directory() +{ + if (!IS_EXRTO_READ) { + return; + } + if (mkdir(EXRTO_FILE_DIR, S_IRWXU) < 0 && errno != EEXIST) { + ereport(ERROR, (errcode_for_file_access(), errmsg("could not create directory \"%s\": %m", EXRTO_FILE_DIR))); + } + + char sub_dir[EXRTO_FILE_PATH_LEN]; + errno_t rc = EOK; + for (ExRTOFileType type = BASE_PAGE; type <= BLOCK_INFO_META; type = static_cast(type + 1)) { + rc = snprintf_s(sub_dir, EXRTO_FILE_PATH_LEN, EXRTO_FILE_PATH_LEN - 1, "%s/%s", EXRTO_FILE_DIR, + EXRTO_FILE_SUB_DIR[type]); + securec_check_ss(rc, "\0", "\0"); + if (mkdir(sub_dir, S_IRWXU) < 0 && errno != EEXIST) { + ereport(ERROR, (errcode_for_file_access(), errmsg("could not create directory \"%s\": %m", sub_dir))); + } + } +} + +void exrto_clean_dir(void) +{ + int ret = 0; + ereport(LOG, (errmsg("exrto_clean_dir: start to clean dir."))); + if (!isDirExist(EXRTO_FILE_DIR)) { + return; + } + + if (!isDirExist(EXRTO_OLD_FILE_DIR)) { + ereport(LOG, (errmsg("exrto_clean_dir: rename standby_read to standby_read_old."))); + ret = rename(EXRTO_FILE_DIR, EXRTO_OLD_FILE_DIR); + if (ret != 0) { + ereport(ERROR, (errcode_for_file_access(), + errmsg("failed to rename exrto standby_read dir: %s\n", EXRTO_FILE_DIR))); + return; + } + } else { + ereport(LOG, (errmsg("exrto_clean_dir: remove standby_read."))); + if (!rmtree(EXRTO_FILE_DIR, true)) { + ereport(WARNING, (errcode_for_file_access(), + errmsg("could not remove exrto standby_read dir: %s\n", EXRTO_FILE_DIR))); + } + } +} + +/* This function will be attached to the recycle thread */ +void exrto_recycle_old_dir(void) +{ + if (!rmtree(EXRTO_OLD_FILE_DIR, true)) { + ereport(WARNING, (errcode_for_file_access(), + errmsg("could not remove exrto standby_read_old dir: %s\n", EXRTO_OLD_FILE_DIR))); + } +} + +void exrto_standby_read_init() +{ + exrto_clean_dir(); + if (IS_EXRTO_READ) { + make_exrto_file_directory(); + } +} + +Datum gs_hot_standby_space_info(PG_FUNCTION_ARGS) +{ +#define EXRTO_HOT_STANDBY_SPACE_INFO_INFONUM 6 + Datum values[EXRTO_HOT_STANDBY_SPACE_INFO_INFONUM]; + errno_t rc; + bool nulls[EXRTO_HOT_STANDBY_SPACE_INFO_INFONUM]; + HeapTuple tuple = NULL; + TupleDesc tupdesc = NULL; + uint64 lsn_file_size = 0; + uint64 lsn_file_num = 0; + uint64 basepage_file_size = 0; + uint64 basepage_file_num = 0; + uint64 block_meta_file_size = 0; + uint64 block_meta_file_num = 0; + uint32 worker_nums; + + rc = memset_s(values, sizeof(values), 0, sizeof(values)); + securec_check(rc, "\0", "\0"); + + rc = memset_s(nulls, sizeof(nulls), 0, sizeof(nulls)); + securec_check(rc, "\0", "\0"); + + tupdesc = CreateTemplateTupleDesc(EXRTO_HOT_STANDBY_SPACE_INFO_INFONUM, false); + TupleDescInitEntry(tupdesc, (AttrNumber)ARG_1, "base_page_file_num", XIDOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)ARG_2, "base_page_total_size", XIDOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)ARG_3, "lsn_info_meta_file_num", XIDOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)ARG_4, "lsn_info_meta_total_size", XIDOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)ARG_5, "block_info_meta_file_num", XIDOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)ARG_6, "block_info_meta_total_size", XIDOID, -1, 0); + + tupdesc = BlessTupleDesc(tupdesc); + + SpinLockAcquire(&(g_instance.comm_cxt.predo_cxt.destroy_lock)); + if (extreme_rto::g_dispatcher == NULL) { + worker_nums = 0; + } else { + worker_nums = extreme_rto::g_dispatcher->allWorkersCnt; + } + + for (uint32 i = 0; i < worker_nums; ++i) { + extreme_rto::PageRedoWorker* page_redo_worker = extreme_rto::g_dispatcher->allWorkers[i]; + if (page_redo_worker->role != extreme_rto::REDO_PAGE_WORKER) { + continue; + } + StandbyReadMetaInfo meta_info = page_redo_worker->standby_read_meta_info; + + uint64 lsn_file_size_per_thread = 0; + if (meta_info.lsn_table_next_position > meta_info.lsn_table_recyle_position) { + lsn_file_size_per_thread = meta_info.lsn_table_next_position - meta_info.lsn_table_recyle_position; + /* in 0~lsn_table_recyle_position No data is stored, + means the size of one lsn info file does not reach maxsize + eg:0~100KB(lsn_table_recyle_position), 100KB~(16M+100KB)(lsn_table_next_position), filenum:2, size:16M */ + lsn_file_num += meta_info.lsn_table_next_position / EXRTO_LSN_INFO_FILE_MAXSIZE + + ((meta_info.lsn_table_next_position % EXRTO_LSN_INFO_FILE_MAXSIZE) > 0 ? 1 : 0) - + (meta_info.lsn_table_recyle_position / EXRTO_LSN_INFO_FILE_MAXSIZE); + } + lsn_file_size += lsn_file_size_per_thread; + + uint64 basepage_file_size_per_thread = 0; + if (meta_info.base_page_next_position > meta_info.base_page_recyle_position) { + basepage_file_size_per_thread = meta_info.base_page_next_position - meta_info.base_page_recyle_position; + basepage_file_num += meta_info.base_page_next_position / EXRTO_BASE_PAGE_FILE_MAXSIZE + + ((meta_info.base_page_next_position % EXRTO_BASE_PAGE_FILE_MAXSIZE) > 0 ? 1 : 0) - + (meta_info.base_page_recyle_position / EXRTO_BASE_PAGE_FILE_MAXSIZE); + } + basepage_file_size += basepage_file_size_per_thread; + } + SpinLockRelease(&(g_instance.comm_cxt.predo_cxt.destroy_lock)); + + char block_meta_file_dir[EXRTO_FILE_PATH_LEN]; + char block_meta_file_name[EXRTO_FILE_PATH_LEN]; + struct dirent *de = NULL; + struct stat st; + + rc = snprintf_s(block_meta_file_dir, EXRTO_FILE_PATH_LEN, EXRTO_FILE_PATH_LEN - 1, "./%s/%s", + EXRTO_FILE_DIR, EXRTO_BLOCK_INFO_SUB_DIR); + securec_check_ss(rc, "\0", "\0"); + + DIR *dir = opendir(block_meta_file_dir); + while ((dir != NULL) && (de = gs_readdir(dir)) != NULL) { + if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) { + continue; + } + rc = snprintf_s(block_meta_file_name, EXRTO_FILE_PATH_LEN, EXRTO_FILE_PATH_LEN - 1, "%s/%s", + block_meta_file_dir, de->d_name); + securec_check_ss(rc, "\0", "\0"); + if (lstat(block_meta_file_name, &st) != 0) { + continue; + } + block_meta_file_num++; + block_meta_file_size = block_meta_file_size + (uint64)st.st_size; + } + + values[ARG_0] = TransactionIdGetDatum(basepage_file_num); + values[ARG_1] = TransactionIdGetDatum(basepage_file_size); + values[ARG_2] = TransactionIdGetDatum(lsn_file_num); + values[ARG_3] = TransactionIdGetDatum(lsn_file_size); + values[ARG_4] = TransactionIdGetDatum(block_meta_file_num); + values[ARG_5] = TransactionIdGetDatum(block_meta_file_size); + + tuple = heap_form_tuple(tupdesc, values, nulls); + PG_RETURN_DATUM(HeapTupleGetDatum(tuple)); +} + diff --git a/src/gausskernel/storage/access/transam/extreme_rto/Makefile b/src/gausskernel/storage/access/transam/extreme_rto/Makefile index 6b4b4968e..06e25e75f 100644 --- a/src/gausskernel/storage/access/transam/extreme_rto/Makefile +++ b/src/gausskernel/storage/access/transam/extreme_rto/Makefile @@ -26,6 +26,6 @@ top_builddir = ../../../../../.. include $(top_builddir)/src/Makefile.global OBJS = dispatcher.o page_redo.o posix_semaphore.o redo_item.o \ - spsc_blocking_queue.o txn_redo.o batch_redo.o xlog_read.o + spsc_blocking_queue.o txn_redo.o batch_redo.o xlog_read.o exrto_recycle.o include $(top_srcdir)/src/gausskernel/common.mk diff --git a/src/gausskernel/storage/access/transam/extreme_rto/batch_redo.cpp b/src/gausskernel/storage/access/transam/extreme_rto/batch_redo.cpp index e45dff0b6..c81eb9f5e 100644 --- a/src/gausskernel/storage/access/transam/extreme_rto/batch_redo.cpp +++ b/src/gausskernel/storage/access/transam/extreme_rto/batch_redo.cpp @@ -44,25 +44,6 @@ #include "access/xlogproc.h" namespace extreme_rto { -static inline void PRXLogRecGetBlockTag(XLogRecParseState *recordBlockState, RelFileNode *rnode, BlockNumber *blknum, - ForkNumber *forknum) -{ - XLogBlockParse *blockparse = &(recordBlockState->blockparse); - - if (rnode != NULL) { - rnode->dbNode = blockparse->blockhead.dbNode; - rnode->relNode = blockparse->blockhead.relNode; - rnode->spcNode = blockparse->blockhead.spcNode; - rnode->bucketNode = blockparse->blockhead.bucketNode; - rnode->opt = blockparse->blockhead.opt; - } - if (blknum != NULL) { - *blknum = blockparse->blockhead.blkno; - } - if (forknum != NULL) { - *forknum = blockparse->blockhead.forknum; - } -} void PRInitRedoItemEntry(RedoItemHashEntry *redoItemHashEntry) { diff --git a/src/gausskernel/storage/access/transam/extreme_rto/dispatcher.cpp b/src/gausskernel/storage/access/transam/extreme_rto/dispatcher.cpp index 9a1daeb2b..61e853c84 100755 --- a/src/gausskernel/storage/access/transam/extreme_rto/dispatcher.cpp +++ b/src/gausskernel/storage/access/transam/extreme_rto/dispatcher.cpp @@ -391,7 +391,7 @@ void SSAllocRecordReadBuffer(XLogReaderState *xlogreader, uint32 privateLen) #endif } -void HandleStartupInterruptsForExtremeRto() +void StartupInterruptsForExtremeRto() { Assert(AmStartupProcess()); @@ -400,7 +400,7 @@ void HandleStartupInterruptsForExtremeRto() uint32 triggeredstate = pg_atomic_read_u32(&(g_startupTriggerState)); if (triggeredstate != newtriggered) { ereport(LOG, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), - errmsg("HandleStartupInterruptsForExtremeRto:g_startupTriggerState set from %u to %u", + errmsg("StartupInterruptsForExtremeRto:g_startupTriggerState set from %u to %u", triggeredstate, newtriggered))); pg_atomic_write_u32(&(g_startupTriggerState), newtriggered); } @@ -452,9 +452,15 @@ void StartRecoveryWorkers(XLogReaderState *xlogreader, uint32 privateLen) SpinLockAcquire(&(g_instance.comm_cxt.predo_cxt.rwlock)); g_instance.comm_cxt.predo_cxt.state = REDO_IN_PROGRESS; SpinLockRelease(&(g_instance.comm_cxt.predo_cxt.rwlock)); + + Assert(g_instance.pid_cxt.exrto_recycler_pid == 0); + if (g_instance.attr.attr_storage.EnableHotStandby) { + g_instance.pid_cxt.exrto_recycler_pid = initialize_util_thread(EXRTO_RECYCLER); + } + on_shmem_exit(StopRecoveryWorkers, 0); - g_dispatcher->oldStartupIntrruptFunc = RegisterRedoInterruptCallBack(HandleStartupInterruptsForExtremeRto); + g_dispatcher->oldStartupIntrruptFunc = RegisterRedoInterruptCallBack(StartupInterruptsForExtremeRto); close_readFile_if_open(); } @@ -563,6 +569,9 @@ static void StartPageRedoWorkers(uint32 totalThrdNum) for (uint32 j = 0; j < batchWorkerPerMng; j++) { RedoRoleInit(&(g_dispatcher->pageLines[i].redoThd[j]), tmpWorkers[workerCnt++], REDO_PAGE_WORKER, j, isUndoSpaceWorker); + // start from 1 not 0 + g_dispatcher->pageLines[i].redoThd[j]->standby_read_meta_info.batch_id = i + 1; + g_dispatcher->pageLines[i].redoThd[j]->standby_read_meta_info.redo_id = j + 1; } g_dispatcher->pageLines[i].redoThdNum = batchWorkerPerMng; } @@ -607,6 +616,10 @@ bool DispathCouldExit() } } + if (g_instance.pid_cxt.exrto_recycler_pid != 0) { + return false; + } + return true; } @@ -636,6 +649,17 @@ void SendSingalToPageWorker(int signal) } } +void send_signal_to_eros_recycle_worker(int signal) +{ + if (g_instance.pid_cxt.exrto_recycler_pid != 0) { + int err = gs_signal_send(g_instance.pid_cxt.exrto_recycler_pid, signal); + if (err != 0) { + ereport(WARNING, (errmsg("Dispatch kill(pid %lu, signal %d) failed: \"%s\",", + g_instance.pid_cxt.exrto_recycler_pid, signal, gs_strerror(err)))); + } + } +} + /* Run from the dispatcher thread. */ static void StopRecoveryWorkers(int code, Datum arg) { @@ -643,6 +667,7 @@ static void StopRecoveryWorkers(int code, Datum arg) errmsg("parallel redo workers are going to stop, code:%d, arg:%lu", code, DatumGetUInt64(arg)))); SendSingalToPageWorker(SIGTERM); + send_signal_to_eros_recycle_worker(SIGTERM); uint64 count = 0; while (!DispathCouldExit()) { @@ -1125,6 +1150,7 @@ static bool DispatchDataBaseRecord(XLogReaderState *record, List *expectedTLIs, if (IsDataBaseDrop(record)) { isNeedFullSync = true; + record->isFullSync = true; RedoItem *item = GetRedoItemPtr(record); ReferenceRedoItem(item); @@ -1132,10 +1158,18 @@ static bool DispatchDataBaseRecord(XLogReaderState *record, List *expectedTLIs, ReferenceRedoItem(item); AddPageRedoItem(g_dispatcher->pageLines[i].batchThd, item); } - DereferenceRedoItem(item); + AddTxnRedoItem(g_dispatcher->trxnLine.managerThd, item); } else { /* database dir may impact many rel so need to sync to all pageworks */ - DispatchRecordWithoutPage(record, expectedTLIs); + record->isFullSync = true; + RedoItem *item = GetRedoItemPtr(record); + + ReferenceRedoItem(item); + for (uint32 i = 0; i < g_dispatcher->pageLineNum; i++) { + ReferenceRedoItem(item); + AddPageRedoItem(g_dispatcher->pageLines[i].batchThd, item); + } + AddTxnRedoItem(g_dispatcher->trxnLine.managerThd, item); g_dispatcher->needFullSyncCheckpoint = true; } @@ -1911,6 +1945,7 @@ void SendRecoveryEndMarkToWorkersAndWaitForFinish(int code) errmsg("[REDO_LOG_TRACE]SendRecoveryEndMarkToWorkersAndWaitForFinish, ready to stop redo workers, code: %d", code))); if ((get_real_recovery_parallelism() > 1) && (GetBatchCount() > 0)) { + send_signal_to_eros_recycle_worker(SIGTERM); WaitPageRedoWorkerReachLastMark(g_dispatcher->readLine.readPageThd); PageRedoPipeline *pl = g_dispatcher->pageLines; /* send end mark */ diff --git a/src/gausskernel/storage/access/transam/extreme_rto/exrto_recycle.cpp b/src/gausskernel/storage/access/transam/extreme_rto/exrto_recycle.cpp new file mode 100644 index 000000000..30f94c2a3 --- /dev/null +++ b/src/gausskernel/storage/access/transam/extreme_rto/exrto_recycle.cpp @@ -0,0 +1,235 @@ +/* + * Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * exrto_recycle.cpp + * + * clean thread for standby read on block level page redo + * + * IDENTIFICATION + * src/gausskernel/storage/access/transam/extreme_rto/exrto_recycle.cpp + * + * ------------------------------------------------------------------------- + */ + +#include "access/extreme_rto/page_redo.h" +#include "access/extreme_rto/dispatcher.h" +#include "access/extreme_rto/standby_read/lsn_info_meta.h" +#include "access/multi_redo_api.h" +#include "storage/ipc.h" +#include "storage/smgr/smgr.h" +#include "utils/memutils.h" + +namespace extreme_rto { +static void exrto_recycle_sighup_handler(SIGNAL_ARGS) +{ + int save_errno = errno; + t_thrd.exrto_recycle_cxt.got_SIGHUP = true; + if (t_thrd.proc) + SetLatch(&t_thrd.proc->procLatch); + errno = save_errno; +} + +static void exrto_recycle_shutdown_handler(SIGNAL_ARGS) +{ + int save_errno = errno; + t_thrd.exrto_recycle_cxt.shutdown_requested = true; + if (t_thrd.proc) { + SetLatch(&t_thrd.proc->procLatch); + } + errno = save_errno; +} + +static void exrto_recycle_quick_die(SIGNAL_ARGS) +{ + int status = 2; + gs_signal_setmask(&t_thrd.libpq_cxt.BlockSig, NULL); + on_exit_reset(); + proc_exit(status); +} + +static void exrto_recycle_setup_signal_handlers() +{ + (void)gspqsignal(SIGHUP, exrto_recycle_sighup_handler); + (void)gspqsignal(SIGINT, SIG_IGN); + (void)gspqsignal(SIGTERM, exrto_recycle_shutdown_handler); + (void)gspqsignal(SIGQUIT, exrto_recycle_quick_die); + (void)gspqsignal(SIGALRM, SIG_IGN); + (void)gspqsignal(SIGPIPE, SIG_IGN); + (void)gspqsignal(SIGUSR1, SIG_IGN); + (void)gspqsignal(SIGUSR2, SIG_IGN); + (void)gspqsignal(SIGCHLD, SIG_IGN); + (void)gspqsignal(SIGTTIN, SIG_IGN); + (void)gspqsignal(SIGTTOU, SIG_IGN); + (void)gspqsignal(SIGCONT, SIG_IGN); + (void)gspqsignal(SIGWINCH, SIG_IGN); + + gs_signal_setmask(&t_thrd.libpq_cxt.UnBlockSig, NULL); + (void)gs_signal_unblock_sigusr2(); +} + +static void handle_exrto_recycle_shutdown() +{ + ereport(LOG, (errmsg("exrto recycle exit for request"))); + ResourceOwnerRelease(t_thrd.utils_cxt.CurrentResourceOwner, RESOURCE_RELEASE_BEFORE_LOCKS, false, true); + proc_exit(0); +} + +static void exrto_recycle_wait() +{ + int rc = 0; + rc = WaitLatch(&t_thrd.proc->procLatch, WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, 1000L); /* 1s */ + /* Clear any already-pending wakeups */ + ResetLatch(&t_thrd.proc->procLatch); + if (((unsigned int)rc) & WL_POSTMASTER_DEATH) { + gs_thread_exit(1); + } +} + +bool check_if_need_force_recycle() +{ + uint32 worker_nums = g_dispatcher->allWorkersCnt; + PageRedoWorker** workers = g_dispatcher->allWorkers; + int64 total_base_page_size = 0; + int64 total_lsn_info_size = 0; + double ratio = g_instance.attr.attr_storage.standby_force_recyle_ratio; + + for (uint32 i = 0; i < worker_nums; ++i) { + PageRedoWorker* page_redo_worker = workers[i]; + StandbyReadMetaInfo meta_info = page_redo_worker->standby_read_meta_info; + if (page_redo_worker->role != REDO_PAGE_WORKER) { + continue; + } + total_base_page_size += (meta_info.base_page_next_position - meta_info.base_page_recyle_position); + total_lsn_info_size += (meta_info.lsn_table_next_position - meta_info.lsn_table_recyle_position); + } + + if (total_base_page_size > g_instance.attr.attr_storage.max_standby_base_page_size * ratio || + total_lsn_info_size > g_instance.attr.attr_storage.max_standby_lsn_info_size * ratio) { + return true; + } + + return false; +} + +void do_standby_read_recyle(XLogRecPtr recycle_lsn) +{ + uint32 worker_nums = g_dispatcher->allWorkersCnt; + PageRedoWorker** workers = g_dispatcher->allWorkers; + XLogRecPtr min_recycle_lsn = InvalidXLogRecPtr; + for (uint32 i = 0; i < worker_nums; ++i) { + PageRedoWorker* page_redo_worker = workers[i]; + if (page_redo_worker->role != REDO_PAGE_WORKER) { + continue; + } + extreme_rto_standby_read::standby_read_recyle_per_workers(&page_redo_worker->standby_read_meta_info, recycle_lsn); + if (XLogRecPtrIsInvalid(min_recycle_lsn) || + XLByteLT(page_redo_worker->standby_read_meta_info.recycle_lsn_per_worker, min_recycle_lsn)) { + min_recycle_lsn = page_redo_worker->standby_read_meta_info.recycle_lsn_per_worker; + } + } + if (XLByteLT(g_instance.comm_cxt.predo_cxt.global_recycle_lsn, min_recycle_lsn)) { + pg_atomic_write_u64(&g_instance.comm_cxt.predo_cxt.global_recycle_lsn, min_recycle_lsn); + ereport(LOG, + (errmsg(EXRTOFORMAT("[exrto_recycle] update global recycle lsn: %08X/%08X"), + (uint32)(min_recycle_lsn >> UINT64_HALF), (uint32)min_recycle_lsn))); + } +} + +void exrto_recycle_interrupt() +{ + if (t_thrd.exrto_recycle_cxt.got_SIGHUP) { + t_thrd.exrto_recycle_cxt.got_SIGHUP = false; + ProcessConfigFile(PGC_SIGHUP); + } + + if (t_thrd.exrto_recycle_cxt.shutdown_requested) { + handle_exrto_recycle_shutdown(); + } +} + +void exrto_recycle_main() +{ + t_thrd.utils_cxt.CurrentResourceOwner = ResourceOwnerCreate(NULL, "exrto recycler", + THREAD_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_STORAGE)); + MemoryContext exrto_recycle_context = AllocSetContextCreate(t_thrd.top_mem_cxt, + "Exrto Recycler", + ALLOCSET_DEFAULT_MINSIZE, + ALLOCSET_DEFAULT_INITSIZE, + ALLOCSET_DEFAULT_MAXSIZE); + (void)MemoryContextSwitchTo(exrto_recycle_context); + + ereport(LOG, (errmsg("exrto recycle started"))); + exrto_recycle_setup_signal_handlers(); + + /* + * Unblock signals (they were blocked when the postmaster forked us) + */ + gs_signal_setmask(&t_thrd.libpq_cxt.UnBlockSig, NULL); + (void)gs_signal_unblock_sigusr2(); + + pgstat_report_appname("exrto recycler"); + pgstat_report_activity(STATE_IDLE, NULL); + + bool need_force_recyle = false; + int sleep_count = 0; + RegisterRedoInterruptCallBack(exrto_recycle_interrupt); + + if (pmState == PM_RUN && isDirExist(EXRTO_FILE_DIR)) { + buffer_drop_exrto_standby_read_buffers(); + exrto_clean_dir(); + } + if (isDirExist(EXRTO_OLD_FILE_DIR)) { + exrto_recycle_old_dir(); + ereport(LOG, (errmsg("exrto recycle: clear standby_read_old dir success"))); + } else { + ereport(LOG, (errmsg("exrto recycle: standby_read_old dir not exist"))); + } + + if (!IS_EXRTO_READ || !RecoveryInProgress()) { + ereport(LOG, + (errmsg("exrto recycle is available only when exrto standby read is supported"))); + handle_exrto_recycle_shutdown(); + } + while (true) { + RedoInterruptCallBack(); + exrto_recycle_wait(); + ++sleep_count; + + /* + * standby_recycle_interval = 0 means do not recyle + */ + if (g_instance.attr.attr_storage.standby_recycle_interval == 0) { + continue; + } + + need_force_recyle = check_if_need_force_recycle(); + if (!need_force_recyle && sleep_count < g_instance.attr.attr_storage.standby_recycle_interval) { + continue; + } + + sleep_count = 0; + + XLogRecPtr recycle_lsn = exrto_calculate_recycle_position(need_force_recyle); + if (XLogRecPtrIsInvalid(recycle_lsn)) { + continue; + } + + do_standby_read_recyle(recycle_lsn); + smgrcloseall(); + MemoryContextResetAndDeleteChildren(exrto_recycle_context); + } + handle_exrto_recycle_shutdown(); +} +} /* namespace extreme_rto */ \ No newline at end of file diff --git a/src/gausskernel/storage/access/transam/extreme_rto/page_redo.cpp b/src/gausskernel/storage/access/transam/extreme_rto/page_redo.cpp index e2b656aaa..24f8a54f0 100755 --- a/src/gausskernel/storage/access/transam/extreme_rto/page_redo.cpp +++ b/src/gausskernel/storage/access/transam/extreme_rto/page_redo.cpp @@ -51,6 +51,7 @@ #include "storage/smgr/relfilenode_hash.h" #include "storage/standby.h" #include "storage/pmsignal.h" +#include "storage/procarray.h" #include "utils/guc.h" #include "utils/palloc.h" #include "portability/instr_time.h" @@ -63,6 +64,7 @@ #include "commands/tablespace.h" #include "access/extreme_rto/page_redo.h" #include "access/extreme_rto/dispatcher.h" +#include "access/extreme_rto/standby_read/lsn_info_meta.h" #include "access/extreme_rto/txn_redo.h" #include "access/extreme_rto/xlog_read.h" #include "pgstat.h" @@ -183,6 +185,9 @@ void RedoWorkerQueueCallBack() bool RedoWorkerIsUndoSpaceWorker() { + if (g_redoWorker == NULL) { + return false; + } return g_redoWorker->isUndoSpaceWorker; } @@ -562,8 +567,6 @@ bool BatchRedoDistributeItems(void **eleArry, uint32 eleNum) BatchRedoProcLsnForwarder((RedoItem *)eleArry[i]); } else if (eleArry[i] == (void *)&g_cleanupMark) { BatchRedoProcCleanupMark((RedoItem *)eleArry[i]); - } else if (eleArry[i] == (void *)&g_closefdMark) { - smgrcloseall(); } else if (eleArry[i] == (void *)&g_cleanInvalidPageMark) { forget_range_invalid_pages((void *)eleArry[i]); } else { @@ -639,26 +642,21 @@ void RedoPageManagerDistributeToAllOneBlock(XLogRecParseState *ddlParseState) } } -void RedoPageManagerDistributeBlockRecord(HTAB *redoItemHash, XLogRecParseState *parsestate) +void RedoPageManagerDistributeBlockRecord(XLogRecParseState *record_block_state) { PageRedoPipeline *myRedoLine = &g_dispatcher->pageLines[g_redoWorker->slotId]; const uint32 WorkerNumPerMng = myRedoLine->redoThdNum; - HASH_SEQ_STATUS status; - RedoItemHashEntry *redoItemEntry = NULL; - HTAB *curMap = redoItemHash; - hash_seq_init(&status, curMap); - - while ((redoItemEntry = (RedoItemHashEntry *)hash_seq_search(&status)) != NULL) { - uint32 workId = GetWorkerId(&redoItemEntry->redoItemTag, WorkerNumPerMng); - AddPageRedoItem(myRedoLine->redoThd[workId], redoItemEntry->head); + uint32 work_id; + RelFileNode rel_node; + ForkNumber fork_num; + BlockNumber blk_no; + RedoItemTag redo_item_tag; - if (hash_search(curMap, (void *)&redoItemEntry->redoItemTag, HASH_REMOVE, NULL) == NULL) - ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("hash table corrupted"))); - } - - if (parsestate != NULL) { - RedoPageManagerDistributeToAllOneBlock(parsestate); - } + PRXLogRecGetBlockTag(record_block_state, &rel_node, &blk_no, &fork_num); + INIT_REDO_ITEM_TAG(redo_item_tag, rel_node, fork_num, blk_no); + work_id = GetWorkerId(&redo_item_tag, WorkerNumPerMng); + record_block_state->nextrecord = NULL; + AddPageRedoItem(myRedoLine->redoThd[work_id], record_block_state); } void WaitCurrentPipeLineRedoWorkersQueueEmpty() @@ -762,13 +760,24 @@ void RedoPageManagerSyncDdlAction(XLogRecParseState *parsestate) XLogBlockParseStateRelease(parsestate); } -void RedoPageManagerDoDropAction(XLogRecParseState *parsestate, HTAB *hashMap) +void RedoPageManagerDoDatabaseAction(XLogRecParseState *parsestate) { - XLogRecParseState *newState = XLogParseBufferCopy(parsestate); - PRTrackClearBlock(newState, hashMap); - RedoPageManagerDistributeBlockRecord(hashMap, parsestate); + RedoPageManagerDistributeToAllOneBlock(parsestate); WaitCurrentPipeLineRedoWorkersQueueEmpty(); - RedoPageManagerSyncDdlAction(parsestate); + RedoPageManagerSmgrClose(parsestate); + + bool need_wait = parsestate->isFullSync; + if (need_wait) { + pg_atomic_write_u32(&g_redoWorker->fullSyncFlag, 1); + } + parsestate->nextrecord = NULL; + XLogBlockParseStateRelease(parsestate); + + uint32 val = pg_atomic_read_u32(&g_redoWorker->fullSyncFlag); + while (val != 0) { + RedoInterruptCallBack(); + val = pg_atomic_read_u32(&g_redoWorker->fullSyncFlag); + } } void RedoPageManagerDoSmgrAction(XLogRecParseState *recordblockstate) @@ -790,16 +799,14 @@ void RedoPageManagerDoSmgrAction(XLogRecParseState *recordblockstate) XLogBlockParseStateRelease(recordblockstate); } -void RedoPageManagerDoDataTypeAction(XLogRecParseState *parsestate, HTAB *hashMap) +void RedoPageManagerDoDataTypeAction(XLogRecParseState *parsestate) { XLogBlockDdlParse *ddlrecparse = NULL; XLogBlockParseGetDdlParse(parsestate, ddlrecparse); if (ddlrecparse->blockddltype == BLOCK_DDL_DROP_RELNODE || ddlrecparse->blockddltype == BLOCK_DDL_TRUNCATE_RELNODE) { - XLogRecParseState *newState = XLogParseBufferCopy(parsestate); - PRTrackClearBlock(newState, hashMap); - RedoPageManagerDistributeBlockRecord(hashMap, parsestate); + RedoPageManagerDistributeToAllOneBlock(parsestate); WaitCurrentPipeLineRedoWorkersQueueEmpty(); } @@ -839,10 +846,10 @@ void PageManagerProcCleanupMark(RedoItem *cleanupMark) ereport(LOG, (errcode(ERRCODE_LOG), errmsg("[ForceFinish]PageManagerProcCleanupMark has cleaned InvalidPages"))); } -void PageManagerProcCheckPoint(HTAB *hashMap, XLogRecParseState *parseState) +void PageManagerProcCheckPoint(XLogRecParseState *parseState) { Assert(IsCheckPoint(parseState)); - RedoPageManagerDistributeBlockRecord(hashMap, parseState); + RedoPageManagerDistributeToAllOneBlock(parseState); bool needWait = parseState->isFullSync; if (needWait) { pg_atomic_write_u32(&g_redoWorker->fullSyncFlag, 1); @@ -865,9 +872,8 @@ void PageManagerProcCheckPoint(HTAB *hashMap, XLogRecParseState *parseState) } } -void PageManagerProcCreateTableSpace(HTAB *hashMap, XLogRecParseState *parseState) +void PageManagerProcCreateTableSpace(XLogRecParseState *parseState) { - RedoPageManagerDistributeBlockRecord(hashMap, NULL); bool needWait = parseState->isFullSync; if (needWait) { pg_atomic_write_u32(&g_redoWorker->fullSyncFlag, 1); @@ -881,16 +887,14 @@ void PageManagerProcCreateTableSpace(HTAB *hashMap, XLogRecParseState *parseStat } } -void PageManagerProcSegFullSyncState(HTAB *hashMap, XLogRecParseState *parseState) +void PageManagerProcSegFullSyncState(XLogRecParseState *parseState) { - RedoPageManagerDistributeBlockRecord(hashMap, NULL); WaitCurrentPipeLineRedoWorkersQueueEmpty(); RedoPageManagerSyncDdlAction(parseState); } -void PageManagerProcSegPipeLineSyncState(HTAB *hashMap, XLogRecParseState *parseState) +void PageManagerProcSegPipeLineSyncState(XLogRecParseState *parseState) { - RedoPageManagerDistributeBlockRecord(hashMap, NULL); WaitCurrentPipeLineRedoWorkersQueueEmpty(); MemoryContext oldCtx = MemoryContextSwitchTo(g_redoWorker->oldCtx); @@ -917,40 +921,38 @@ static void WaitNextBarrier(XLogRecParseState *parseState) void PageManagerRedoParseState(XLogRecParseState *preState) { - HTAB *hashMap = g_dispatcher->pageLines[g_redoWorker->slotId].managerThd->redoItemHash; - switch (preState->blockparse.blockhead.block_valid) { case BLOCK_DATA_MAIN_DATA_TYPE: case BLOCK_DATA_UNDO_TYPE: case BLOCK_DATA_VM_TYPE: case BLOCK_DATA_FSM_TYPE: GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_3]); - PRTrackAddBlock(preState, hashMap); + RedoPageManagerDistributeBlockRecord(preState); CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_3]); break; case BLOCK_DATA_DDL_TYPE: GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_4]); - RedoPageManagerDoDataTypeAction(preState, hashMap); + RedoPageManagerDoDataTypeAction(preState); CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_4]); break; case BLOCK_DATA_SEG_EXTEND: GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_4]); - PageManagerProcSegPipeLineSyncState(hashMap, preState); + PageManagerProcSegPipeLineSyncState(preState); CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_4]); break; + case BLOCK_DATA_CREATE_DATABASE_TYPE: case BLOCK_DATA_DROP_DATABASE_TYPE: GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_5]); - RedoPageManagerDoDropAction(preState, hashMap); + RedoPageManagerDoDatabaseAction(preState); CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_5]); break; case BLOCK_DATA_DROP_TBLSPC_TYPE: /* just make sure any other ddl before drop tblspc is done */ XLogBlockParseStateRelease(preState); break; - case BLOCK_DATA_CREATE_DATABASE_TYPE: case BLOCK_DATA_SEG_FILE_EXTEND_TYPE: GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); - RedoPageManagerDistributeBlockRecord(hashMap, NULL); + RedoPageManagerDistributeBlockRecord(NULL); /* wait until queue empty */ WaitCurrentPipeLineRedoWorkersQueueEmpty(); /* do atcual action */ @@ -959,31 +961,30 @@ void PageManagerRedoParseState(XLogRecParseState *preState) break; case BLOCK_DATA_SEG_FULL_SYNC_TYPE: GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_8]); - PageManagerProcSegFullSyncState(hashMap, preState); + PageManagerProcSegFullSyncState(preState); CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_8]); break; case BLOCK_DATA_CREATE_TBLSPC_TYPE: GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_7]); - PageManagerProcCreateTableSpace(hashMap, preState); + PageManagerProcCreateTableSpace(preState); CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_7]); break; case BLOCK_DATA_XLOG_COMMON_TYPE: - PageManagerProcCheckPoint(hashMap, preState); + PageManagerProcCheckPoint(preState); break; case BLOCK_DATA_NEWCU_TYPE: - RedoPageManagerDistributeBlockRecord(hashMap, NULL); PageManagerDistributeBcmBlock(preState); break; case BLOCK_DATA_SEG_SPACE_DROP: case BLOCK_DATA_SEG_SPACE_SHRINK: GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_8]); - RedoPageManagerDistributeBlockRecord(hashMap, preState); + RedoPageManagerDistributeToAllOneBlock(preState); WaitCurrentPipeLineRedoWorkersQueueEmpty(); RedoPageManagerSyncDdlAction(preState); CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_8]); break; case BLOCK_DATA_BARRIER_TYPE: - RedoPageManagerDistributeBlockRecord(hashMap, preState); + RedoPageManagerDistributeToAllOneBlock(preState); WaitNextBarrier(preState); break; default: @@ -992,71 +993,60 @@ void PageManagerRedoParseState(XLogRecParseState *preState) } } -bool PageManagerRedoDistributeItems(void **eleArry, uint32 eleNum) +bool PageManagerRedoDistributeItems(XLogRecParseState *record_block_state) { - HTAB *hashMap = g_dispatcher->pageLines[g_redoWorker->slotId].managerThd->redoItemHash; + if (record_block_state == (void *)&g_redoEndMark) { + return true; + } else if (record_block_state == (void *)&g_GlobalLsnForwarder) { + PageManagerProcLsnForwarder((RedoItem *) record_block_state); + return false; + } else if (record_block_state == (void *)&g_cleanupMark) { + PageManagerProcCleanupMark((RedoItem *) record_block_state); + return false; + } else if (record_block_state == (void *)&g_cleanInvalidPageMark) { + forget_range_invalid_pages((void *)record_block_state); + return false; + } - for (uint32 i = 0; i < eleNum; i++) { - if (eleArry[i] == (void *)&g_redoEndMark) { - RedoPageManagerDistributeBlockRecord(hashMap, NULL); - return true; - } else if (eleArry[i] == (void *)&g_GlobalLsnForwarder) { - RedoPageManagerDistributeBlockRecord(hashMap, NULL); - PageManagerProcLsnForwarder((RedoItem *)eleArry[i]); - continue; - } else if (eleArry[i] == (void *)&g_cleanupMark) { - PageManagerProcCleanupMark((RedoItem *)eleArry[i]); - continue; - } else if (eleArry[i] == (void *)&g_closefdMark) { - smgrcloseall(); - continue; - } else if (eleArry[i] == (void *)&g_cleanInvalidPageMark) { - forget_range_invalid_pages((void *)eleArry[i]); - continue; - } - XLogRecParseState *recordblockstate = (XLogRecParseState *)eleArry[i]; - XLogRecParseState *nextState = recordblockstate; - do { - XLogRecParseState *preState = nextState; - nextState = (XLogRecParseState *)nextState->nextrecord; - preState->nextrecord = NULL; + XLogRecParseState *next_state = record_block_state; + do { + XLogRecParseState *pre_state = next_state; + next_state = (XLogRecParseState *)next_state->nextrecord; + pre_state->nextrecord = NULL; #ifdef ENABLE_UT - TestXLogRecParseStateEventProbe(UTEST_EVENT_RTO_PAGEMGR_REDO_BEFORE_DISTRIBUTE_ITEMS, - __FUNCTION__, preState); + TestXLogRecParseStateEventProbe(UTEST_EVENT_RTO_PAGEMGR_REDO_BEFORE_DISTRIBUTE_ITEMS, + __FUNCTION__, pre_state); #endif - PageManagerRedoParseState(preState); + PageManagerRedoParseState(pre_state); #ifdef ENABLE_UT - TestXLogRecParseStateEventProbe(UTEST_EVENT_RTO_PAGEMGR_REDO_AFTER_DISTRIBUTE_ITEMS, - __FUNCTION__, preState); + TestXLogRecParseStateEventProbe(UTEST_EVENT_RTO_PAGEMGR_REDO_AFTER_DISTRIBUTE_ITEMS, + __FUNCTION__, pre_state); #endif - } while (nextState != NULL); - } - GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_9]); - RedoPageManagerDistributeBlockRecord(hashMap, NULL); - CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_9]); + } while (next_state != NULL); + return false; } void RedoPageManagerMain() { - void **eleArry; - uint32 eleNum; + XLogRecParseState *record_block_state; + bool is_end; (void)RegisterRedoInterruptCallBack(HandlePageRedoInterrupts); - g_redoWorker->redoItemHash = PRRedoItemHashInitialize(g_redoWorker->oldCtx); XLogParseBufferInitFunc(&(g_redoWorker->parseManager), MAX_PARSE_BUFF_NUM, &recordRefOperate, RedoInterruptCallBack); GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_1]); - while (SPSCBlockingQueueGetAll(g_redoWorker->queue, &eleArry, &eleNum)) { - CountAndGetRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_1], g_redoWorker->timeCostList[TIME_COST_STEP_2]); - bool isEnd = PageManagerRedoDistributeItems(eleArry, eleNum); - SPSCBlockingQueuePopN(g_redoWorker->queue, eleNum); - CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_2]); - if (isEnd) - break; - + while (true) { + if (!SPSCBlockingQueueIsEmpty(g_redoWorker->queue)) { + record_block_state = (XLogRecParseState *)SPSCBlockingQueueTake(g_redoWorker->queue); + CountAndGetRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_1], g_redoWorker->timeCostList[TIME_COST_STEP_2]); + is_end = PageManagerRedoDistributeItems(record_block_state); + CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_2]); + if (is_end) + break; + } RedoInterruptCallBack(); ADD_ABNORMAL_POSITION(5); GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_1]); @@ -1105,7 +1095,8 @@ bool TrxnManagerDistributeItemsBeforeEnd(RedoItem *item) } else { GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_4]); if (IsCheckPoint(&item->record) || IsTableSpaceDrop(&item->record) || IsTableSpaceCreate(&item->record) || - (IsXactXlog(&item->record) && XactWillRemoveRelFiles(&item->record)) || IsBarrierRelated(&item->record)) { + (IsXactXlog(&item->record) && XactWillRemoveRelFiles(&item->record)) || IsBarrierRelated(&item->record) || + IsDataBaseDrop(&item->record) || IsDataBaseCreate(&item->record)) { uint32 relCount; do { RedoInterruptCallBack(); @@ -1200,6 +1191,11 @@ void TrxnManagerMain() void TrxnWorkerProcLsnForwarder(RedoItem *lsnForwarder) { SetCompletedReadEndPtr(g_redoWorker, lsnForwarder->record.ReadRecPtr, lsnForwarder->record.EndRecPtr); + uint32 refcout = pg_atomic_read_u32(&lsnForwarder->record.refcount); + while (refcout > 1) { + refcout = pg_atomic_read_u32(&lsnForwarder->record.refcount); + RedoInterruptCallBack(); + } (void)pg_atomic_sub_fetch_u32(&lsnForwarder->record.refcount, 1); } @@ -1258,13 +1254,11 @@ void TrxnWorkMain() if ((void *)item == (void *)&g_GlobalLsnForwarder) { TrxnWorkerProcLsnForwarder((RedoItem *)item); SPSCBlockingQueuePop(g_redoWorker->queue); - } else if ((void *)item == (void *)&g_cleanupMark) { + exrto_generate_snapshot(g_redoWorker->lastReplayedReadRecPtr); + } else if (unlikely((void *)item == (void *)&g_cleanupMark)) { TrxnWorkrProcCleanupMark((RedoItem *)item); SPSCBlockingQueuePop(g_redoWorker->queue); - } else if ((void *)item == (void *)&g_closefdMark) { - smgrcloseall(); - SPSCBlockingQueuePop(g_redoWorker->queue); - } else if ((void *)item == (void *)&g_cleanInvalidPageMark) { + } else if (unlikely((void *)item == (void *)&g_cleanInvalidPageMark)) { forget_range_invalid_pages((void *)item); SPSCBlockingQueuePop(g_redoWorker->queue); } else { @@ -1283,6 +1277,12 @@ void TrxnWorkMain() TrxnWorkNotifyRedoWorker(); } + if (IsCheckPoint(&item->record) || (IsXactXlog(&item->record) && + XactWillRemoveRelFiles(&item->record)) || IsBarrierRelated(&item->record) || + IsDataBaseDrop(&item->record)) { + exrto_generate_snapshot(g_redoWorker->lastReplayedEndRecPtr); + } + if (XactHasSegpageRelFiles(&item->record)) { uint32 expected = 1; pg_atomic_compare_exchange_u32((volatile uint32 *)&(g_dispatcher->segpageXactDoneFlag), &expected, 0); @@ -1415,12 +1415,6 @@ void RedoPageWorkerMain() continue; } - if ((void *)redoblockstateHead == (void *)&g_closefdMark) { - smgrcloseall(); - SPSCBlockingQueuePop(g_redoWorker->queue); - continue; - } - if ((void *)redoblockstateHead == (void *)&g_cleanInvalidPageMark) { forget_range_invalid_pages((void *)redoblockstateHead); SPSCBlockingQueuePop(g_redoWorker->queue); @@ -1761,26 +1755,26 @@ void DispatchCleanupMarkToAllRedoWorker() } } -void DispatchClosefdMarkToAllRedoWorker() +void DispatchCleanInvalidPageMarkToAllRedoWorker(RepairFileKey key) { for (uint32 i = 0; i < g_dispatcher->allWorkersCnt; i++) { PageRedoWorker *worker = g_dispatcher->allWorkers[i]; - if (worker->role == REDO_PAGE_WORKER || worker->role == REDO_PAGE_MNG || - worker->role == REDO_TRXN_MNG || worker->role == REDO_TRXN_WORKER) { - SPSCBlockingQueuePut(worker->queue, &g_closefdMark); + if (worker->role == REDO_PAGE_WORKER) { + errno_t rc = memcpy_s((char*)&g_cleanInvalidPageMark, + sizeof(RepairFileKey), (char*)&key, sizeof(RepairFileKey)); + securec_check(rc, "", ""); + SPSCBlockingQueuePut(worker->queue, &g_cleanInvalidPageMark); } } } -void DispatchCleanInvalidPageMarkToAllRedoWorker(RepairFileKey key) +void DispatchClosefdMarkToAllRedoWorker() { for (uint32 i = 0; i < g_dispatcher->allWorkersCnt; i++) { PageRedoWorker *worker = g_dispatcher->allWorkers[i]; - if (worker->role == REDO_PAGE_WORKER) { - errno_t rc = memcpy_s((char*)&g_cleanInvalidPageMark, - sizeof(RepairFileKey), (char*)&key, sizeof(RepairFileKey)); - securec_check(rc, "", ""); - SPSCBlockingQueuePut(worker->queue, &g_cleanInvalidPageMark); + if (worker->role == REDO_PAGE_WORKER || worker->role == REDO_PAGE_MNG || + worker->role == REDO_TRXN_MNG || worker->role == REDO_TRXN_WORKER) { + SPSCBlockingQueuePut(worker->queue, &g_closefdMark); } } } @@ -2492,6 +2486,7 @@ void ParallelRedoThreadMain() ParallelRedoThreadRegister(); ereport(LOG, (errmsg("Page-redo-worker thread %u started, role:%u, slotId:%u.", g_redoWorker->id, g_redoWorker->role, g_redoWorker->slotId))); + t_thrd.page_redo_cxt.redo_worker_ptr = g_redoWorker; // regitster default interrupt call back (void)RegisterRedoInterruptCallBack(HandlePageRedoInterrupts); SetupSignalHandlers(); @@ -3011,4 +3006,328 @@ void SeqCheckRemoteReadAndRepairPage() } } +void exrto_generate_snapshot(XLogRecPtr trxn_lsn) +{ + if (!g_instance.attr.attr_storage.EnableHotStandby) { + return; + } + + ExrtoSnapshot exrto_snapshot = &g_dispatcher->exrto_snapshot; + /* + * do not generate the same snapshot repeatedly. + */ + if (XLByteLE(trxn_lsn, exrto_snapshot->read_lsn)) { + return; + } + + if (XLogRecPtrIsInvalid(t_thrd.xlog_cxt.minRecoveryPoint)) { + return; + } + if (XLByteLT(trxn_lsn, exrto_snapshot->read_lsn)) { + return; + } + + SnapshotData snapshot; + + (void)GetSnapshotData(&snapshot, false); + + Assert(snapshot.takenDuringRecovery); + (void)LWLockAcquire(ExrtoSnapshotLock, LW_EXCLUSIVE); + exrto_snapshot->snapshot_csn = snapshot.snapshotcsn; + exrto_snapshot->xmin = snapshot.xmin; + exrto_snapshot->xmax = snapshot.xmax; + exrto_snapshot->read_lsn = trxn_lsn; + exrto_snapshot->gen_snap_time = GetCurrentTimestamp(); + LWLockRelease(ExrtoSnapshotLock); +} + +void exrto_read_snapshot(Snapshot snapshot) +{ + if (t_thrd.role != WORKER && t_thrd.role != THREADPOOL_WORKER) { + return; + } + + if (g_dispatcher == NULL) { + ereport(ERROR, + (errmsg("g_dispatcher is not init")));; + } + + ExrtoSnapshot exrto_snapshot = &g_dispatcher->exrto_snapshot; + (void)LWLockAcquire(ExrtoSnapshotLock, LW_SHARED); + if (XLByteEQ(exrto_snapshot->read_lsn, 0)) { + LWLockRelease(ExrtoSnapshotLock); + ereport(ERROR, + (errmsg("could not get a valid snapshot with extreme rto"))); + } + snapshot->snapshotcsn = exrto_snapshot->snapshot_csn; + snapshot->xmin = exrto_snapshot->xmin; + snapshot->xmax = exrto_snapshot->xmax; + + t_thrd.pgxact->xmin = exrto_snapshot->xmin; + t_thrd.proc->exrto_read_lsn = exrto_snapshot->read_lsn; + t_thrd.proc->exrto_gen_snap_time = exrto_snapshot->gen_snap_time; + u_sess->utils_cxt.TransactionXmin = exrto_snapshot->xmin; + u_sess->utils_cxt.exrto_read_lsn = exrto_snapshot->read_lsn; + + LWLockRelease(ExrtoSnapshotLock); + Assert(XLogRecPtrIsValid(t_thrd.proc->exrto_read_lsn)); +} + +static inline uint64 get_force_recycle_pos(uint64 recycle_pos, uint64 insert_pos) +{ + const double force_recyle_ratio = 0.3; /* to be adjusted */ + Assert(recycle_pos <= insert_pos); + return recycle_pos + (uint64)((insert_pos - recycle_pos) * force_recyle_ratio); +} + +XLogRecPtr calculate_force_recycle_lsn_per_worker(StandbyReadMetaInfo* meta_info) +{ + uint64 base_page_recycle_pos; + uint64 lsn_info_recycle_pos; + XLogRecPtr base_page_recycle_lsn = InvalidXLogRecPtr; + XLogRecPtr lsn_info_recycle_lsn = InvalidXLogRecPtr; + Buffer buffer; + Page page; + + /* for base page */ + if (meta_info->base_page_recyle_position < meta_info->base_page_next_position) { + base_page_recycle_pos = get_force_recycle_pos(meta_info->base_page_recyle_position, + meta_info->base_page_next_position); + buffer = extreme_rto_standby_read::buffer_read_base_page(meta_info->batch_id, meta_info->redo_id, + base_page_recycle_pos, RBM_NORMAL); + LockBuffer(buffer, BUFFER_LOCK_SHARE); + base_page_recycle_lsn = PageGetLSN(BufferGetPage(buffer)); + UnlockReleaseBuffer(buffer); + } + + /* for lsn info */ + if (meta_info->lsn_table_recyle_position < meta_info->lsn_table_next_position) { + lsn_info_recycle_pos = get_force_recycle_pos(meta_info->lsn_table_recyle_position, + meta_info->lsn_table_next_position); + page = extreme_rto_standby_read::get_lsn_info_page(meta_info->batch_id, meta_info->redo_id, + lsn_info_recycle_pos, RBM_NORMAL, &buffer); + if (unlikely(page == NULL || buffer == InvalidBuffer)) { + ereport(PANIC, + (errmsg(EXRTOFORMAT("get_lsn_info_page failed, batch_id: %u, redo_id: %u, pos: %lu"), + meta_info->batch_id, meta_info->redo_id, lsn_info_recycle_pos))); + } + LockBuffer(buffer, BUFFER_LOCK_SHARE); + extreme_rto_standby_read::LsnInfo lsn_info = (extreme_rto_standby_read::LsnInfo)(page + + extreme_rto_standby_read::LSN_INFO_HEAD_SIZE); + lsn_info_recycle_lsn = lsn_info->lsn[0]; + UnlockReleaseBuffer(buffer); + } + + return rtl::max(base_page_recycle_lsn, lsn_info_recycle_lsn); +} + +void calculate_force_recycle_lsn(XLogRecPtr &recycle_lsn) +{ + XLogRecPtr recycle_lsn_per_worker; + uint32 worker_nums = g_dispatcher->allWorkersCnt; + PageRedoWorker** workers = g_dispatcher->allWorkers; + + for (uint32 i = 0; i < worker_nums; ++i) { + PageRedoWorker* page_redo_worker = workers[i]; + if (page_redo_worker->role != REDO_PAGE_WORKER) { + continue; + } + recycle_lsn_per_worker = calculate_force_recycle_lsn_per_worker(&page_redo_worker->standby_read_meta_info); + if (XLByteLT(recycle_lsn, recycle_lsn_per_worker)) { + recycle_lsn = recycle_lsn_per_worker; + } + } + ereport(LOG, + (errmsg(EXRTOFORMAT("[exrto_recycle] try force recycle, recycle lsn: %08X/%08X"), + (uint32)(recycle_lsn >> UINT64_HALF), (uint32)recycle_lsn))); +} + +static inline bool exceed_standby_max_query_time(TimestampTz start_time) +{ + return TimestampDifferenceExceeds(start_time, GetCurrentTimestamp(), + g_instance.attr.attr_storage.standby_max_query_time * MSECS_PER_SEC); +} + +/* 1. resolve recycle conflict with backends + * 2. get oldest xmin and oldest readlsn of backends. */ +void proc_array_get_oldeset_readlsn(XLogRecPtr recycle_lsn, XLogRecPtr &oldest_lsn, TransactionId &oldest_xmin, + bool &conflict) +{ + ProcArrayStruct* proc_array = g_instance.proc_array_idx; + conflict = false; + + LWLockAcquire(ProcArrayLock, LW_SHARED); + for (int index = 0; index < proc_array->numProcs; index++) { + int pg_proc_no = proc_array->pgprocnos[index]; + PGPROC* pg_proc = g_instance.proc_base_all_procs[pg_proc_no]; + PGXACT* pg_xact = &g_instance.proc_base_all_xacts[pg_proc_no]; + XLogRecPtr read_lsn = pg_proc->exrto_read_lsn; + TransactionId pxmin = pg_xact->xmin; + + if (pg_proc->pid == 0 || !TransactionIdIsValid(pxmin) || XLogRecPtrIsInvalid(read_lsn)) { + continue; + } + + Assert(!(pg_xact->vacuumFlags & PROC_IN_VACUUM)); + /* + * Backend is doing logical decoding which manages xmin + * separately, check below. + */ + if (pg_xact->vacuumFlags & PROC_IN_LOGICAL_DECODING) { + continue; + } + + /* cancel query when its read_lsn < recycle_lsn or its runtime > standby_max_query_time */ + if (XLByteLT(read_lsn, recycle_lsn) || exceed_standby_max_query_time(pg_proc->exrto_gen_snap_time)) { + pg_proc->recoveryConflictPending = true; + conflict = true; + if (pg_proc->pid != 0) { + /* + * Kill the pid if it's still here. If not, that's what we + * wanted so ignore any errors. + */ + (void)SendProcSignal(pg_proc->pid, PROCSIG_RECOVERY_CONFLICT_SNAPSHOT, pg_proc->backendId); + /* + * Wait a little bit for it to die so that we avoid flooding + * an unresponsive backend when system is heavily loaded. + */ + pg_usleep(5000L); + } + continue; + } + + if (XLogRecPtrIsInvalid(oldest_lsn) || + (XLogRecPtrIsValid(read_lsn) && XLByteLT(read_lsn, oldest_lsn))) { + oldest_lsn = read_lsn; + } + + if (!TransactionIdIsValid(oldest_xmin) || + (TransactionIdIsValid(pxmin) && TransactionIdFollows(oldest_xmin, pxmin))) { + oldest_xmin = pxmin; + } + } + LWLockRelease(ProcArrayLock); +} + +void proc_array_get_oldeset_xmin_for_undo(TransactionId &oldest_xmin) +{ + ProcArrayStruct* proc_array = g_instance.proc_array_idx; + + LWLockAcquire(ProcArrayLock, LW_SHARED); + for (int index = 0; index < proc_array->numProcs; index++) { + int pg_proc_no = proc_array->pgprocnos[index]; + PGPROC* pg_proc = g_instance.proc_base_all_procs[pg_proc_no]; + PGXACT* pg_xact = &g_instance.proc_base_all_xacts[pg_proc_no]; + TransactionId pxmin = pg_xact->xmin; + + if (pg_proc->pid == 0 || !TransactionIdIsValid(pxmin)) { + continue; + } + + Assert(!(pg_xact->vacuumFlags & PROC_IN_VACUUM)); + /* + * Backend is doing logical decoding which manages xmin + * separately, check below. + */ + if (pg_xact->vacuumFlags & PROC_IN_LOGICAL_DECODING) { + continue; + } + if (!TransactionIdIsValid(oldest_xmin) || + (TransactionIdIsValid(pxmin) && TransactionIdFollows(oldest_xmin, pxmin))) { + oldest_xmin = pxmin; + } + } + LWLockRelease(ProcArrayLock); +} + +XLogRecPtr exrto_calculate_recycle_position(bool force_recyle) +{ + Assert(t_thrd.role != PAGEREDO); + Assert(IS_EXRTO_READ); + + XLogRecPtr recycle_lsn = g_instance.comm_cxt.predo_cxt.global_recycle_lsn; + XLogRecPtr oldest_lsn = InvalidXLogRecPtr; + TransactionId oldest_xmin = InvalidTransactionId; + bool conflict = false; + const int max_check_times = 1000; + int check_times = 0; + + if (force_recyle) { + calculate_force_recycle_lsn(recycle_lsn); + } + + /* Loop checks to avoid conflicting queries that were not successfully canceled. */ + do { + RedoInterruptCallBack(); + proc_array_get_oldeset_readlsn(recycle_lsn, oldest_lsn, oldest_xmin, conflict); + check_times++; + } while (conflict && check_times < max_check_times); + + /* + * If there is no backend read threads, set read oldest lsn to snapshot lsn. + */ + if (XLogRecPtrIsInvalid(oldest_lsn)) { + ExrtoSnapshot exrto_snapshot = NULL; + exrto_snapshot = &g_dispatcher->exrto_snapshot; + (void)LWLockAcquire(ExrtoSnapshotLock, LW_SHARED); + if (XLByteEQ(exrto_snapshot->read_lsn, 0)) { + ereport(WARNING, + (errmsg("could not get a valid snapshot with extreme rto"))); + } else { + oldest_lsn = exrto_snapshot->read_lsn; + oldest_xmin = exrto_snapshot->xmin; + } + + LWLockRelease(ExrtoSnapshotLock); + } + recycle_lsn = rtl::max(recycle_lsn, oldest_lsn); + + ereport( + LOG, + (errmsg( + EXRTOFORMAT( + "[exrto_recycle] calculate recycle position, oldestlsn: %08X/%08X, snapshot read_lsn: %08X/%08X, try " + "recycle lsn: %08X/%08X"), + (uint32)(oldest_lsn >> UINT64_HALF), (uint32)oldest_lsn, + (uint32)(g_dispatcher->exrto_snapshot.read_lsn >> UINT64_HALF), + (uint32)g_dispatcher->exrto_snapshot.read_lsn, (uint32)(recycle_lsn >> UINT64_HALF), (uint32)recycle_lsn))); + + return recycle_lsn; +} + +TransactionId exrto_calculate_recycle_xmin_for_undo() +{ + Assert(t_thrd.role != PAGEREDO); + Assert(IS_EXRTO_READ); + TransactionId oldest_xmin = InvalidTransactionId; + TransactionId snapshot_xmin = InvalidTransactionId; + proc_array_get_oldeset_xmin_for_undo(oldest_xmin); + + /* + * If there is no backend read threads, set read oldest lsn to snapshot lsn. + */ + if (oldest_xmin == InvalidTransactionId) { + ExrtoSnapshot exrto_snapshot = NULL; + exrto_snapshot = &g_dispatcher->exrto_snapshot; + (void)LWLockAcquire(ExrtoSnapshotLock, LW_SHARED); + if (XLByteEQ(exrto_snapshot->xmin, InvalidTransactionId)) { + ereport( + WARNING, + (errmsg("exrto_calculate_recycle_xmin_for_undo: could not get a valid snapshot in exrto_snapshot"))); + } else { + snapshot_xmin = exrto_snapshot->xmin; + } + + LWLockRelease(ExrtoSnapshotLock); + } + ereport(DEBUG1, (errmodule(MOD_UNDO), + errmsg(UNDOFORMAT("exrto_calculate_recycle_xmin_for_undo: oldest_xmin: %lu, snapshot_xmin: %lu."), + oldest_xmin, snapshot_xmin))); + + if (oldest_xmin == InvalidTransactionId) { + return snapshot_xmin; + } + return oldest_xmin; +} + } // namespace extreme_rto \ No newline at end of file diff --git a/src/gausskernel/storage/access/transam/multi_redo_api.cpp b/src/gausskernel/storage/access/transam/multi_redo_api.cpp index 2d70a75a6..7b2b56444 100644 --- a/src/gausskernel/storage/access/transam/multi_redo_api.cpp +++ b/src/gausskernel/storage/access/transam/multi_redo_api.cpp @@ -66,9 +66,9 @@ void DispatchRedoRecord(XLogReaderState *record, List *expectedTLIs, TimestampTz g_instance.comm_cxt.localinfo_cxt.term_from_xlog = term; } - long readbufcountbefore = u_sess->instr_cxt.pg_buffer_usage->local_blks_read; + long readbufcountbefore = u_sess->instr_cxt.pg_buffer_usage->shared_blks_read; ApplyRedoRecord(record); - record->readblocks = u_sess->instr_cxt.pg_buffer_usage->local_blks_read - readbufcountbefore; + record->readblocks = u_sess->instr_cxt.pg_buffer_usage->shared_blks_read - readbufcountbefore; CountXLogNumbers(record); if (XLogRecGetRmid(record) == RM_XACT_ID) SetLatestXTime(recordXTime); @@ -134,6 +134,10 @@ bool IsAllPageWorkerExit() } g_instance.comm_cxt.predo_cxt.totalNum = 0; } + + if (g_instance.pid_cxt.exrto_recycler_pid != 0) { + return false; + } ereport(LOG, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), errmsg("page workers all exit or not open parallel redo"))); diff --git a/src/gausskernel/storage/access/transam/xact.cpp b/src/gausskernel/storage/access/transam/xact.cpp index aae577d6e..68e632abb 100755 --- a/src/gausskernel/storage/access/transam/xact.cpp +++ b/src/gausskernel/storage/access/transam/xact.cpp @@ -46,6 +46,7 @@ #include "access/xloginsert.h" #include "access/xlogutils.h" #include "access/multi_redo_api.h" +#include "access/extreme_rto/standby_read/block_info_meta.h" #include "catalog/catalog.h" #include "catalog/namespace.h" #include "catalog/pg_authid.h" @@ -7206,6 +7207,15 @@ static void unlink_relfiles(_in_ ColFileNode *xnodes, _in_ int nrels) smgrdounlink(srel, true); smgrclose(srel); + /* + * recycle exrto files when dropping table occurs. + */ + if (IS_EXRTO_READ) { + RelFileNode block_meta_file = relFileNode; + block_meta_file.spcNode = EXRTO_BLOCK_INFO_SPACE_OID; + extreme_rto_standby_read::remove_one_block_info_file(block_meta_file); + } + UnlockRelFileNode(relFileNode, AccessExclusiveLock); /* diff --git a/src/gausskernel/storage/access/transam/xlog.cpp b/src/gausskernel/storage/access/transam/xlog.cpp index 3a85240b5..28fe66427 100755 --- a/src/gausskernel/storage/access/transam/xlog.cpp +++ b/src/gausskernel/storage/access/transam/xlog.cpp @@ -38,6 +38,7 @@ #include "access/double_write.h" #include "access/heapam.h" #include "access/multixact.h" +#include "access/multi_redo_api.h" #include "access/rewriteheap.h" #include "access/subtrans.h" #include "access/transam.h" @@ -10424,6 +10425,12 @@ void StartupXLOG(void) } else { Insert->fullPageWrites = t_thrd.xlog_cxt.lastFullPageWrites; } + + if (IS_EXRTO_READ) { + /* we are going to be master, we need to recycle residual_undo_file again */ + g_instance.undo_cxt.is_exrto_residual_undo_file_recycled = false; + } + LocalSetXLogInsertAllowed(); UpdateFullPageWrites(); t_thrd.xlog_cxt.LocalXLogInsertAllowed = -1; @@ -13182,6 +13189,16 @@ static void KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo, XLogRecPtr curIns segno = mainStandbySegNo; } } + + if (IS_EXRTO_READ) { + XLogRecPtr recycle_recptr = pg_atomic_read_u64(&g_instance.comm_cxt.predo_cxt.global_recycle_lsn); + XLogSegNo recyle_segno; + XLByteToSeg(recycle_recptr, recyle_segno); + if (recyle_segno < segno && recyle_segno > 0) { + segno = recyle_segno; + } + } + /* don't delete WAL segments newer than the calculated segment */ if (segno < *logSegNo && segno > 0) { *logSegNo = segno; @@ -16019,8 +16036,8 @@ void SetXLogReplayRecPtr(XLogRecPtr readRecPtr, XLogRecPtr endRecPtr) SpinLockRelease(&xlogctl->info_lck); if (isUpdated) { RedoSpeedDiag(readRecPtr, endRecPtr); + update_dirty_page_queue_rec_lsn(readRecPtr); } - update_dirty_page_queue_rec_lsn(readRecPtr); #ifndef ENABLE_MULTIPLE_NODES if (g_instance.attr.attr_storage.dcf_attr.enable_dcf) { int ret = dcf_set_election_priority(1, endRecPtr); diff --git a/src/gausskernel/storage/access/ustore/knl_uextremeredo.cpp b/src/gausskernel/storage/access/ustore/knl_uextremeredo.cpp index 6e70f321f..9149870fe 100644 --- a/src/gausskernel/storage/access/ustore/knl_uextremeredo.cpp +++ b/src/gausskernel/storage/access/ustore/knl_uextremeredo.cpp @@ -1148,9 +1148,6 @@ void UHeapXlogFreezeTDOperatorPage(RedoBufferInfo *buffer, void *recorddata) UHeapPageTDData *tdPtr = (UHeapPageTDData *)PageGetTDPointer(page); TD *transinfo = tdPtr->td_info; - if (InHotStandby && TransactionIdIsValid(xlrec->latestFrozenXid)) - ResolveRecoveryConflictWithSnapshot(xlrec->latestFrozenXid, buffer->blockinfo.rnode, buffer->lsn); - UHeapFreezeOrInvalidateTuples(buffer->buf, nFrozen, frozenSlots, true); for (int i = 0; i < nFrozen; i++) { @@ -1520,12 +1517,6 @@ void UHeapRedoDataBlock(XLogBlockHead *blockhead, XLogBlockDataParse *blockdatar } } -#ifdef ENABLE_MULTIPLE_NODES -const static bool SUPPORT_HOT_STANDBY = false; /* don't support consistency view */ -#else -const static bool SUPPORT_HOT_STANDBY = true; -#endif - void UHeap2XlogFreezeOperatorPage(RedoBufferInfo *buffer, void *recorddata, void *blkdata, Size datalen) { XlUHeapFreeze *xlrec = (XlUHeapFreeze *)recorddata; @@ -1536,14 +1527,6 @@ void UHeap2XlogFreezeOperatorPage(RedoBufferInfo *buffer, void *recorddata, void OffsetNumber *offsetsEnd = NULL; UHeapTupleData utuple; - /* - * In Hot Standby mode, ensure that there's no queries running which still - * consider the frozen xids as running. - */ - if (InHotStandby && SUPPORT_HOT_STANDBY) { - ResolveRecoveryConflictWithSnapshot(cutoffXid, buffer->blockinfo.rnode, buffer->lsn); - } - if (datalen > 0) { offsetsEnd = (OffsetNumber *)((char *)offsets + datalen); @@ -2019,12 +2002,18 @@ static void RedoUndoDiscardBlock(XLogBlockHead *blockhead, XLogBlockUndoParse *b XLogRecPtr lsn = blockdatarec->undoDiscardParse.lsn; UndoZone *zone = UndoZoneGroup::GetUndoZone(zoneId); + ereport(DEBUG1, (errmodule(MOD_UNDO), errmsg(UNDOFORMAT( + "redo_undo_discard_block zid=%d, isZoneNull:%d, zone_lsn:%lu, lsn:%lu, end_slot:%lu, end_undo_ptr:%lu, " + "recycled_xid:%lu."), zoneId, (int)(zone == NULL), zone->GetLSN(), lsn, endSlot, endUndoPtr, recycledXid))); if (zone == NULL) { return; } if (zone->GetLSN() < lsn) { zone->LockUndoZone(); Assert(blockdatarec->undoDiscardParse.startSlot == zone->GetRecycleTSlotPtr()); + if (IS_EXRTO_READ && (!g_instance.undo_cxt.is_exrto_residual_undo_file_recycled)) { + zone->set_recycle_tslot_ptr_exrto(endSlot); + } zone->SetRecycleTSlotPtr(endSlot); zone->SetDiscardURecPtr(endUndoPtr); zone->SetForceDiscardURecPtr(endUndoPtr); @@ -2048,12 +2037,19 @@ static void RedoUndoUnlinkBlock(XLogBlockHead *blockhead, XLogBlockUndoParse *bl XLogRecPtr unlinkLsn = blockdatarec->undoUnlinkParse.unlinkLsn; UndoLogOffset newHead = blockdatarec->undoUnlinkParse.headOffset; UndoLogOffset head = usp->Head(); + ereport(DEBUG1, (errmodule(MOD_UNDO), errmsg(UNDOFORMAT( + "redo_undo_unlink_block, zid=%d, usp_lsn:%lu, unlink_lsn:%lu, head:%lu, new_head:%lu."), + zoneId, usp->LSN(), unlinkLsn, head, newHead))); if (usp->LSN() < unlinkLsn) { zone->ForgetUndoBuffer(head, newHead, UNDO_DB_OID); usp->LockSpace(); usp->MarkDirty(); - usp->UnlinkUndoLog(zoneId, newHead, UNDO_DB_OID); + if (IS_EXRTO_STANDBY_READ) { + usp->SetHead(newHead); + } else { + usp->UnlinkUndoLog(zoneId, newHead, UNDO_DB_OID); + } usp->SetLSN(unlinkLsn); usp->UnlockSpace(); } @@ -2071,12 +2067,19 @@ static void RedoSlotUnlinkBlock(XLogBlockHead *blockhead, XLogBlockUndoParse *bl XLogRecPtr unlinkLsn = blockdatarec->undoUnlinkParse.unlinkLsn; UndoLogOffset newHead = blockdatarec->undoUnlinkParse.headOffset; UndoLogOffset head = usp->Head(); + ereport(DEBUG1, (errmodule(MOD_UNDO), errmsg(UNDOFORMAT( + "redo_slot_unlink_block, zid=%d, usp_lsn:%lu, unlink_lsn:%lu, head:%lu, new_head:%lu."), + zoneId, usp->LSN(), unlinkLsn, head, newHead))); if (usp->LSN() < unlinkLsn) { zone->ForgetUndoBuffer(head, newHead, UNDO_SLOT_DB_OID); usp->LockSpace(); usp->MarkDirty(); - usp->UnlinkUndoLog(zoneId, newHead, UNDO_SLOT_DB_OID); + if (IS_EXRTO_STANDBY_READ) { + usp->SetHead(newHead); + } else { + usp->UnlinkUndoLog(zoneId, newHead, UNDO_SLOT_DB_OID); + } usp->SetLSN(unlinkLsn); usp->UnlockSpace(); } diff --git a/src/gausskernel/storage/access/ustore/undo/knl_uundoapi.cpp b/src/gausskernel/storage/access/ustore/undo/knl_uundoapi.cpp index bb72f6e27..87acf77e5 100644 --- a/src/gausskernel/storage/access/ustore/undo/knl_uundoapi.cpp +++ b/src/gausskernel/storage/access/ustore/undo/knl_uundoapi.cpp @@ -22,6 +22,7 @@ #include "access/ustore/undo/knl_uundoxlog.h" #include "access/ustore/knl_whitebox_test.h" #include "access/transam.h" +#include "access/multi_redo_api.h" #include "catalog/pg_class.h" #include "knl/knl_session.h" #include "knl/knl_thread.h" @@ -281,9 +282,14 @@ UndoRecordState CheckUndoRecordValid(UndoRecPtr urp, bool checkForceRecycle, Tra UndoZone *uzone = UndoZoneGroup::GetUndoZone(zid, false); if (uzone == NULL) { return UNDO_RECORD_INVALID; - } else { - return uzone->CheckUndoRecordValid(UNDO_PTR_GET_OFFSET(urp), checkForceRecycle, lastXid); } + + if (IS_EXRTO_STANDBY_READ) { + return uzone->check_record_valid_exrto(UNDO_PTR_GET_OFFSET(urp), checkForceRecycle, lastXid); + } + + + return uzone->CheckUndoRecordValid(UNDO_PTR_GET_OFFSET(urp), checkForceRecycle, lastXid); } /* @@ -640,6 +646,7 @@ void RecoveryUndoSystemMeta(void) /* Close fd. */ close(fd); + ereport(LOG, (errcode(ERRCODE_DATA_EXCEPTION), errmsg("recovery_meta: undo recovery finish."))); #endif } } diff --git a/src/gausskernel/storage/access/ustore/undo/knl_uundorecycle.cpp b/src/gausskernel/storage/access/ustore/undo/knl_uundorecycle.cpp index 75ac16a32..44debc23e 100755 --- a/src/gausskernel/storage/access/ustore/undo/knl_uundorecycle.cpp +++ b/src/gausskernel/storage/access/ustore/undo/knl_uundorecycle.cpp @@ -31,6 +31,8 @@ #include "access/ustore/undo/knl_uundoxlog.h" #include "access/ustore/knl_undorequest.h" #include "access/ustore/knl_whitebox_test.h" +#include "access/multi_redo_api.h" +#include "access/extreme_rto/page_redo.h" #include "gssignal/gs_signal.h" #include "knl/knl_thread.h" #include "storage/ipc.h" @@ -528,6 +530,162 @@ static void RecycleWaitIfNotUsed() } } +void exrto_standby_release_space(UndoZone *zone, TransactionId recycle_xid, UndoRecPtr start_undo_ptr, + UndoRecPtr end_undo_ptr, UndoSlotPtr recycle_exrto) +{ + UndoRecPtr oldest_end_undo_ptr = end_undo_ptr; + Assert(TransactionIdIsValid(recycle_xid) && (zone->get_recycle_xid_exrto() < recycle_xid)); + zone->LockUndoZone(); + if (!zone->CheckRecycle(start_undo_ptr, end_undo_ptr)) { + ereport(PANIC, (errmodule(MOD_UNDO), + errmsg(UNDOFORMAT("zone %d recycle start %lu >= recycle end %lu."), + zone->GetZoneId(), start_undo_ptr, end_undo_ptr))); + } + if (IS_VALID_UNDO_REC_PTR(oldest_end_undo_ptr)) { + int start_zid = UNDO_PTR_GET_ZONE_ID(start_undo_ptr); + int end_zid = UNDO_PTR_GET_ZONE_ID(oldest_end_undo_ptr); + if (unlikely(start_zid != end_zid)) { + oldest_end_undo_ptr = MAKE_UNDO_PTR(start_zid, UNDO_LOG_MAX_SIZE); + } + zone->set_discard_urec_ptr_exrto(oldest_end_undo_ptr); + } + + zone->set_recycle_xid_exrto(recycle_xid); + zone->set_force_discard_urec_ptr_exrto(end_undo_ptr); + zone->set_recycle_tslot_ptr_exrto(recycle_exrto); + zone->UnlockUndoZone(); + zone->ReleaseSpace(start_undo_ptr, end_undo_ptr, &g_forceRecycleSize); + zone->ReleaseSlotSpace(0, recycle_exrto, &g_forceRecycleSize); +} + +bool exrto_standby_recycle_space(UndoZone *zone, TransactionId recycle_xmin) +{ + UndoSlotPtr recycle_exrto = zone->get_recycle_tslot_ptr_exrto(); + UndoSlotPtr recycle_primary = zone->GetRecycleTSlotPtr(); + undo::TransactionSlot *slot = NULL; + UndoRecPtr end_undo_ptr = INVALID_UNDO_REC_PTR; + TransactionId recycle_xid = InvalidTransactionId; + bool undo_recycled = false; + bool result = false; + UndoSlotPtr start = INVALID_UNDO_SLOT_PTR; + ereport(DEBUG1, (errmodule(MOD_UNDO), + errmsg(UNDOFORMAT("exrto_standby_recycle_space zone_id:%d, recycle_xmin:%lu, recycle_exrto:%lu, " + "recycle_primary:%lu."), + zone->GetZoneId(), recycle_xmin, recycle_exrto, recycle_primary))); + + while (recycle_exrto < recycle_primary) { + UndoSlotBuffer& slot_buf = g_slotBufferCache->FetchTransactionBuffer(recycle_exrto); + UndoRecPtr start_undo_ptr = INVALID_UNDO_REC_PTR; + start = recycle_exrto; + slot_buf.PrepareTransactionSlot(recycle_exrto); + undo_recycled = false; + Assert(slot_buf.BufBlock() == UNDO_PTR_GET_BLOCK_NUM(recycle_exrto)); + while (slot_buf.BufBlock() == UNDO_PTR_GET_BLOCK_NUM(recycle_exrto) && (recycle_exrto < recycle_primary)) { + slot = slot_buf.FetchTransactionSlot(recycle_exrto); + if (!TransactionIdIsValid(slot->XactId())) { + break; + } + if (slot->StartUndoPtr() == INVALID_UNDO_REC_PTR) { + break; + } + + if (TransactionIdFollowsOrEquals(slot->XactId(), recycle_xmin)) { + break; + } + ereport(DEBUG1, (errmodule(MOD_UNDO), + errmsg(UNDOFORMAT("recycle zone %d, exrto transaction slot %lu xid %lu start ptr %lu end ptr %lu."), + zone->GetZoneId(), recycle_exrto, slot->XactId(), + slot->StartUndoPtr(), slot->EndUndoPtr()))); + if (!start_undo_ptr) { + start_undo_ptr = slot->StartUndoPtr(); + } + end_undo_ptr = slot->EndUndoPtr(); + recycle_xid = slot->XactId(); + undo_recycled = true; + recycle_exrto = GetNextSlotPtr(recycle_exrto); + /* if next recycle_exrto is in different slot_buf, release current slot_buf. */ + if (slot_buf.BufBlock() != UNDO_PTR_GET_BLOCK_NUM(recycle_exrto)) { + g_slotBufferCache->RemoveSlotBuffer(start); + slot_buf.Release(); + } + } + if (undo_recycled) { + exrto_standby_release_space(zone, recycle_xid, start_undo_ptr, end_undo_ptr, recycle_exrto); + result = true; + } else { + /* zone has nothing to recycle. */ + break; + } + } + return result; +} + +bool exrto_standby_recycle_undo_zone() +{ + uint32 idx = 0; + bool recycled = false; + if (g_instance.undo_cxt.uZoneCount == 0 || g_instance.undo_cxt.uZones == NULL) { + return recycled; + } + TransactionId recycle_xmin = extreme_rto::exrto_calculate_recycle_xmin_for_undo(); + for (idx = 0; idx < PERSIST_ZONE_COUNT && !t_thrd.undorecycler_cxt.shutdown_requested; idx++) { + UndoZone *zone = (UndoZone *)g_instance.undo_cxt.uZones[idx]; + if (zone == NULL) { + continue; + } + if (zone->Used_exrto()) { + if (exrto_standby_recycle_space(zone, recycle_xmin)) { + recycled = true; + } + } + } + smgrcloseall(); + return recycled; +} + +/* recycle residual_undo_file which may be leftover by exrto read in standby */ +void exrto_recycle_residual_undo_file() +{ + uint32 idx = 0; + uint64 record_file_cnt = 0; + uint64 slot_file_cnt = 0; + if (g_instance.undo_cxt.is_exrto_residual_undo_file_recycled) { + return; + } + ereport(LOG, (errmodule(MOD_UNDO), + errmsg(UNDOFORMAT("exrto_recycle_residual_undo_file begin uZoneCount is %u."), + g_instance.undo_cxt.uZoneCount))); + if (g_instance.undo_cxt.uZoneCount == 0 || g_instance.undo_cxt.uZones == NULL) { + g_instance.undo_cxt.is_exrto_residual_undo_file_recycled = true; + ereport(LOG, (errmodule(MOD_UNDO), + errmsg(UNDOFORMAT("exrto_recycle_residual_undo_file uZoneCount is zero or uZones is null.")))); + return; + } + for (idx = 0; idx < PERSIST_ZONE_COUNT && !t_thrd.undorecycler_cxt.shutdown_requested; idx++) { + UndoZone *zone = (UndoZone *)g_instance.undo_cxt.uZones[idx]; + if (zone == NULL) { + continue; + } + record_file_cnt += zone->release_residual_record_space(); + slot_file_cnt += zone->release_residual_slot_space(); + } + smgrcloseall(); + ereport(LOG, (errmodule(MOD_UNDO), + errmsg(UNDOFORMAT("exrto_recycle_residual_undo_file release record_file_cnt:%lu, " + "slot_file_cnt:%lu."), record_file_cnt, slot_file_cnt))); + g_instance.undo_cxt.is_exrto_residual_undo_file_recycled = true; +} + +void recycle_wait(bool recycled, uint64 *non_recycled) +{ + if (!recycled) { + *non_recycled += UNDO_RECYCLE_TIMEOUT_DELTA; + WaitRecycleThread(*non_recycled); + } else { + *non_recycled = 0; + } +} + void UndoRecycleMain() { sigjmp_buf localSigjmpBuf; @@ -646,6 +804,10 @@ void UndoRecycleMain() t_thrd.undorecycler_cxt.got_SIGHUP = false; ProcessConfigFile(PGC_SIGHUP); } + if (t_thrd.undorecycler_cxt.shutdown_requested) { + ShutDownRecycle(recycleMaxXIDs); + } + exrto_recycle_residual_undo_file(); if (!RecoveryInProgress()) { TransactionId recycleXmin = InvalidTransactionId; TransactionId oldestXmin = GetOldestXminForUndo(&recycleXmin); @@ -751,15 +913,10 @@ void UndoRecycleMain() pg_atomic_write_u64(&g_instance.undo_cxt.globalRecycleXid, oldestXidHavingUndo); } } - if (!recycled) { - nonRecycled += UNDO_RECYCLE_TIMEOUT_DELTA; - WaitRecycleThread(nonRecycled); - } else { - nonRecycled = 0; - } - } else { - WaitRecycleThread(nonRecycled); + } else if (IS_EXRTO_STANDBY_READ) { + recycled = exrto_standby_recycle_undo_zone(); } + recycle_wait(recycled, &nonRecycled); } ShutDownRecycle(recycleMaxXIDs); } diff --git a/src/gausskernel/storage/access/ustore/undo/knl_uundospace.cpp b/src/gausskernel/storage/access/ustore/undo/knl_uundospace.cpp index 24a29e01e..3d60a8a4d 100644 --- a/src/gausskernel/storage/access/ustore/undo/knl_uundospace.cpp +++ b/src/gausskernel/storage/access/ustore/undo/knl_uundospace.cpp @@ -20,6 +20,7 @@ #include "access/ustore/knl_whitebox_test.h" #include "storage/lock/lwlock.h" #include "storage/smgr/smgr.h" +#include "access/multi_redo_api.h" namespace undo { static uint64 USEG_SIZE(uint32 dbId) @@ -49,6 +50,26 @@ uint32 UndoSpace::Used(void) return (uint32)((tail_ - head_) / BLCKSZ); } +UndoLogOffset UndoSpace::find_oldest_offset(int zid, uint32 db_id) const +{ + UndoLogOffset offset = head_; + BlockNumber blockno; + RelFileNode rnode; + UNDO_PTR_ASSIGN_REL_FILE_NODE(rnode, MAKE_UNDO_PTR(zid, offset), db_id); + SMgrRelation reln = smgropen(rnode, InvalidBackendId); + uint64 seg_size = USEG_SIZE(db_id); + while (offset >=seg_size) { + offset -= seg_size; + blockno = (BlockNumber)(offset / BLCKSZ); + if (!smgrexists(reln, MAIN_FORKNUM, blockno)) { + offset += seg_size; + break; + } + } + smgrclose(reln); + return offset; +} + /* Create segments needed to increase end_ to newEnd. */ void UndoSpace::ExtendUndoLog(int zid, UndoLogOffset offset, uint32 dbId) { @@ -91,7 +112,17 @@ void UndoSpace::ExtendUndoLog(int zid, UndoLogOffset offset, uint32 dbId) void UndoSpace::UnlinkUndoLog(int zid, UndoLogOffset offset, uint32 dbId) { RelFileNode rnode; - UndoLogOffset head = head_; + UndoLogOffset head; + UndoLogOffset old_head; + if (IS_EXRTO_STANDBY_READ) { + head = head_exrto; + old_head = head_exrto; + set_head_exrto(offset); + } else { + head = head_; + old_head = head_; + SetHead(offset); + } Assert(head < offset && head_ <= tail_); UNDO_PTR_ASSIGN_REL_FILE_NODE(rnode, MAKE_UNDO_PTR(zid, offset), dbId); SMgrRelation reln = smgropen(rnode, InvalidBackendId); @@ -104,6 +135,9 @@ void UndoSpace::UnlinkUndoLog(int zid, UndoLogOffset offset, uint32 dbId) while (head < offset) { /* Create a new undo segment. */ smgrdounlink(reln, t_thrd.xlog_cxt.InRecovery, (head / BLCKSZ)); + ereport(DEBUG1, (errmodule(MOD_UNDO), errmsg(UNDOFORMAT( + "unlink undo log, zid=%d, dbid=%u, new_head=%lu, segId:%lu."), + zid, dbId, offset, head/segSize))); if (g_instance.undo_cxt.undoTotalSize < segBlocks) { ereport(PANIC, (errmodule(MOD_UNDO), errmsg(UNDOFORMAT( "unlink undo log, total blocks=%u < segment size."), @@ -114,8 +148,32 @@ void UndoSpace::UnlinkUndoLog(int zid, UndoLogOffset offset, uint32 dbId) } smgrclose(reln); ereport(DEBUG1, (errmodule(MOD_UNDO), errmsg(UNDOFORMAT( - "unlink undo log, total blocks=%u, zid=%d, dbid=%u, head=%lu."), - g_instance.undo_cxt.undoTotalSize, zid, dbId, offset))); + "unlink undo log, total blocks=%u, zid=%d, dbid=%u, head=%lu, old_head:%lu."), + g_instance.undo_cxt.undoTotalSize, zid, dbId, offset, old_head))); + return; +} + +/* + * Unlink undo segment files which are residual in extreme RTO standby read, + * unlink from start to end(not include). + */ +void UndoSpace::unlink_residual_log(int zid, UndoLogOffset start, UndoLogOffset end, uint32 db_id) const +{ + RelFileNode rnode; + UNDO_PTR_ASSIGN_REL_FILE_NODE(rnode, MAKE_UNDO_PTR(zid, start), db_id); + SMgrRelation reln = smgropen(rnode, InvalidBackendId); + uint64 seg_size = USEG_SIZE(db_id); + + while (start/seg_size < end/seg_size) { + /* delete a new undo segment. */ + BlockNumber block = (BlockNumber)(start / BLCKSZ); + smgrdounlink(reln, t_thrd.xlog_cxt.InRecovery, block); + ereport(DEBUG1, (errmodule(MOD_UNDO), errmsg(UNDOFORMAT( + "unlink_residual_log, zid=%d, dbid=%u, start=%lu, end=%lu, segId:%lu, endSegId:%lu."), + zid, db_id, start, end, start/seg_size, end/seg_size))); + start += seg_size; + } + smgrclose(reln); return; } @@ -383,6 +441,7 @@ void UndoSpace::RecoveryUndoSpace(int fd, UndoSpaceType type) usp->MarkClean(); usp->SetLSN(uspMetaInfo->lsn); usp->SetHead(uspMetaInfo->head); + usp->set_head_exrto(uspMetaInfo->head); usp->SetTail(uspMetaInfo->tail); if (type == UNDO_LOG_SPACE) { usp->CreateNonExistsUndoFile(zoneId, UNDO_DB_OID); @@ -390,6 +449,9 @@ void UndoSpace::RecoveryUndoSpace(int fd, UndoSpaceType type) usp->CreateNonExistsUndoFile(zoneId, UNDO_SLOT_DB_OID); } pg_atomic_fetch_add_u32(&g_instance.undo_cxt.undoTotalSize, usp->Used()); + ereport(DEBUG1, (errmsg(UNDOFORMAT("recovery_space_meta, zone_id:%u, type:%u, " + "lsn:%lu, head:%lu, tail:%lu."), + zoneId, type, uspMetaInfo->lsn, uspMetaInfo->head, uspMetaInfo->tail))); } pfree(persistBlock); } diff --git a/src/gausskernel/storage/access/ustore/undo/knl_uundozone.cpp b/src/gausskernel/storage/access/ustore/undo/knl_uundozone.cpp index 288f12ad5..7ac14bfd4 100644 --- a/src/gausskernel/storage/access/ustore/undo/knl_uundozone.cpp +++ b/src/gausskernel/storage/access/ustore/undo/knl_uundozone.cpp @@ -21,6 +21,7 @@ #include "access/ustore/undo/knl_uundotxn.h" #include "access/ustore/undo/knl_uundospace.h" #include "access/ustore/knl_whitebox_test.h" +#include "access/multi_redo_api.h" #include "knl/knl_thread.h" #include "miscadmin.h" #include "storage/smgr/fd.h" @@ -43,11 +44,15 @@ UndoZone::UndoZone() SetLSN(0); SetInsertURecPtr(UNDO_LOG_BLOCK_HEADER_SIZE); SetDiscardURecPtr(UNDO_LOG_BLOCK_HEADER_SIZE); + set_discard_urec_ptr_exrto(UNDO_LOG_BLOCK_HEADER_SIZE); SetForceDiscardURecPtr(UNDO_LOG_BLOCK_HEADER_SIZE); + set_force_discard_urec_ptr_exrto(UNDO_LOG_BLOCK_HEADER_SIZE); SetAllocateTSlotPtr(UNDO_LOG_BLOCK_HEADER_SIZE); SetRecycleTSlotPtr(UNDO_LOG_BLOCK_HEADER_SIZE); + set_recycle_tslot_ptr_exrto(UNDO_LOG_BLOCK_HEADER_SIZE); SetFrozenSlotPtr(INVALID_UNDO_SLOT_PTR); SetRecycleXid(InvalidTransactionId); + set_recycle_xid_exrto(InvalidTransactionId); SetFrozenXid(InvalidTransactionId); InitSlotBuffer(); SetAttachPid(0); @@ -56,12 +61,14 @@ UndoZone::UndoZone() undoSpace_.LockInit(); undoSpace_.SetLSN(0); undoSpace_.SetHead(0); + undoSpace_.set_head_exrto(0); undoSpace_.SetTail(0); slotSpace_.MarkClean(); slotSpace_.LockInit(); slotSpace_.SetLSN(0); slotSpace_.SetHead(0); + slotSpace_.set_head_exrto(0); slotSpace_.SetTail(0); } @@ -80,10 +87,16 @@ bool UndoZone::CheckRecycle(UndoRecPtr starturp, UndoRecPtr endurp) int endZid = UNDO_PTR_GET_ZONE_ID(endurp); UndoLogOffset start = UNDO_PTR_GET_OFFSET(starturp); UndoLogOffset end = UNDO_PTR_GET_OFFSET(endurp); - Assert(start == forceDiscardURecPtr_); + UndoLogOffset force_discard_urec_ptr; + if (IS_EXRTO_STANDBY_READ) { + force_discard_urec_ptr = force_discard_urec_ptr_exrto; + } else { + force_discard_urec_ptr = forceDiscardURecPtr_; + } + Assert(start == force_discard_urec_ptr); WHITEBOX_TEST_STUB(UNDO_CHECK_RECYCLE_FAILED, WhiteboxDefaultErrorEmit); - if ((startZid == endZid) && (forceDiscardURecPtr_ <= insertURecPtr_) && (end <= insertURecPtr_) + if ((startZid == endZid) && (force_discard_urec_ptr <= insertURecPtr_) && (end <= insertURecPtr_) && (start < end)) { return true; } @@ -130,6 +143,48 @@ UndoRecordState UndoZone::CheckUndoRecordValid(UndoLogOffset offset, bool checkF return UNDO_RECORD_DISCARD; } +/* + * Check whether the undo record is discarded or not. If it's already discarded + * return false otherwise return true. Caller must hold the space discardLock_. + */ +UndoRecordState UndoZone::check_record_valid_exrto(UndoLogOffset offset, bool check_force_recycle, + TransactionId *last_xid) const +{ + Assert((offset < UNDO_LOG_MAX_SIZE) && (offset >= UNDO_LOG_BLOCK_HEADER_SIZE)); + Assert(force_discard_urec_ptr_exrto <= insertURecPtr_); + + if (offset >= this->insertURecPtr_) { + ereport(DEBUG1, (errmsg(UNDOFORMAT("The undo record not insert yet: zid=%d, " + "insert=%lu, offset=%lu."), + this->zid_, this->insertURecPtr_, offset))); + return UNDO_RECORD_NOT_INSERT; + } + if (offset >= this->force_discard_urec_ptr_exrto) { + return UNDO_RECORD_NORMAL; + } + if (last_xid != NULL) { + *last_xid = recycle_xid_exrto; + } + if (offset >= this->discard_urec_ptr_exrto && check_force_recycle) { + TransactionId recycle_xmin; + TransactionId oldest_xmin = GetOldestXminForUndo(&recycle_xmin); + if (TransactionIdPrecedes(recycle_xid_exrto, recycle_xmin)) { + ereport(DEBUG1, (errmsg( + UNDOFORMAT("oldestxmin %lu, recycle_xmin %lu > recyclexid_exrto %lu: zid=%d," + "force_discard_urec_ptr_exrto=%lu, discard_urec_ptr_exrto=%lu, offset=%lu."), + oldest_xmin, recycle_xmin, recycle_xid_exrto, this->zid_, this->force_discard_urec_ptr_exrto, + this->discard_urec_ptr_exrto, offset))); + return UNDO_RECORD_DISCARD; + } + ereport(DEBUG1, (errmsg(UNDOFORMAT("The record has been force recycled: zid=%d, " + "force_discard_urec_ptr_exrto=%lu, " + "discard_urec_ptr_exrto=%lu, offset=%lu."), + this->zid_, this->force_discard_urec_ptr_exrto, this->discard_urec_ptr_exrto, offset))); + return UNDO_RECORD_FORCE_DISCARD; + } + return UNDO_RECORD_DISCARD; +} + /* * Drop all buffers for the given undo log, from the start to end. */ @@ -220,7 +275,14 @@ UndoSlotPtr UndoZone::AllocateSlotSpace(void) void UndoZone::ReleaseSpace(UndoRecPtr starturp, UndoRecPtr endurp, int *forceRecycleSize) { UndoLogOffset end = UNDO_PTR_GET_OFFSET(endurp); - int startSegno = (int)(undoSpace_.Head() / UNDO_LOG_SEGMENT_SIZE); + int startSegno; + UndoLogOffset head; + if (IS_EXRTO_STANDBY_READ) { + head = undoSpace_.Head_exrto(); + } else { + head = undoSpace_.Head(); + } + startSegno = (int)(head / UNDO_LOG_SEGMENT_SIZE); int endSegno = (int)(end / UNDO_LOG_SEGMENT_SIZE); if (unlikely(startSegno < endSegno)) { @@ -229,10 +291,10 @@ void UndoZone::ReleaseSpace(UndoRecPtr starturp, UndoRecPtr endurp, int *forceRe } ForgetUndoBuffer(startSegno * UNDO_LOG_SEGMENT_SIZE, endSegno * UNDO_LOG_SEGMENT_SIZE, UNDO_DB_OID); undoSpace_.LockSpace(); - UndoRecPtr prevHead = MAKE_UNDO_PTR(zid_, undoSpace_.Head()); + UndoRecPtr prevHead = MAKE_UNDO_PTR(zid_, head); undoSpace_.UnlinkUndoLog(zid_, endSegno * UNDO_LOG_SEGMENT_SIZE, UNDO_DB_OID); Assert(undoSpace_.Head() <= insertURecPtr_); - if (pLevel_ == UNDO_PERMANENT) { + if (pLevel_ == UNDO_PERMANENT && (!IS_EXRTO_STANDBY_READ)) { START_CRIT_SECTION(); undoSpace_.MarkDirty(); XlogUndoUnlink undoUnlink; @@ -247,11 +309,35 @@ void UndoZone::ReleaseSpace(UndoRecPtr starturp, UndoRecPtr endurp, int *forceRe return; } +/* Release undo space from starturp to endurp and advance discard. */ +uint64 UndoZone::release_residual_record_space() +{ + undoSpace_.LockSpace(); + UndoLogOffset unlink_start = undoSpace_.find_oldest_offset(zid_, UNDO_DB_OID); + UndoLogOffset unlink_end = undoSpace_.Head(); + undoSpace_.unlink_residual_log(zid_, unlink_start, unlink_end, UNDO_DB_OID); + undoSpace_.UnlockSpace(); + if (unlink_start > unlink_end) { + ereport(WARNING, (errmsg(UNDOFORMAT("release_residual_record_space start:%lu " + "is bigger than end:%lu."), + unlink_start, unlink_end))); + return 0; + } else { + return (unlink_end / UNDO_LOG_SEGMENT_SIZE) - (unlink_start / UNDO_LOG_SEGMENT_SIZE); + } +} + /* Release slot space from starturp to endurp and advance discard. */ void UndoZone::ReleaseSlotSpace(UndoRecPtr startSlotPtr, UndoRecPtr endSlotPtr, int *forceRecycleSize) { UndoLogOffset end = UNDO_PTR_GET_OFFSET(endSlotPtr); - int startSegno = (int)(slotSpace_.Head() / UNDO_META_SEGMENT_SIZE); + UndoLogOffset head; + if (IS_EXRTO_STANDBY_READ) { + head = slotSpace_.Head_exrto(); + } else { + head = slotSpace_.Head(); + } + int startSegno = (int)(head / UNDO_META_SEGMENT_SIZE); int endSegno = (int)(end / UNDO_META_SEGMENT_SIZE); if (unlikely(startSegno < endSegno)) { @@ -260,10 +346,10 @@ void UndoZone::ReleaseSlotSpace(UndoRecPtr startSlotPtr, UndoRecPtr endSlotPtr, } ForgetUndoBuffer(startSegno * UNDO_META_SEGMENT_SIZE, endSegno * UNDO_META_SEGMENT_SIZE, UNDO_SLOT_DB_OID); slotSpace_.LockSpace(); - UndoRecPtr prevHead = MAKE_UNDO_PTR(zid_, slotSpace_.Head()); + UndoRecPtr prevHead = MAKE_UNDO_PTR(zid_, head); slotSpace_.UnlinkUndoLog(zid_, endSegno * UNDO_META_SEGMENT_SIZE, UNDO_SLOT_DB_OID); Assert(slotSpace_.Head() <= allocateTSlotPtr_); - if (pLevel_ == UNDO_PERMANENT) { + if (pLevel_ == UNDO_PERMANENT && !(IS_EXRTO_STANDBY_READ)) { START_CRIT_SECTION(); slotSpace_.MarkDirty(); XlogUndoUnlink undoUnlink; @@ -278,6 +364,24 @@ void UndoZone::ReleaseSlotSpace(UndoRecPtr startSlotPtr, UndoRecPtr endSlotPtr, return; } +/* Release slot space from starturp to endurp and advance discard. */ +uint64 UndoZone::release_residual_slot_space() +{ + slotSpace_.LockSpace(); + UndoLogOffset unlink_start = slotSpace_.find_oldest_offset(zid_, UNDO_SLOT_DB_OID); + UndoLogOffset unlink_end = slotSpace_.Head(); + slotSpace_.unlink_residual_log(zid_, unlink_start, unlink_end, UNDO_SLOT_DB_OID); + slotSpace_.UnlockSpace(); + if (unlink_start > unlink_end) { + ereport(WARNING, (errmsg(UNDOFORMAT("release_residual_slot_space start:%lu is bigger " + "than end:%lu."), + unlink_start, unlink_end))); + return 0; + } else { + return (unlink_end / UNDO_META_SEGMENT_SIZE) - (unlink_start / UNDO_META_SEGMENT_SIZE); + } +} + void UndoZone::PrepareSwitch(void) { WHITEBOX_TEST_STUB(UNDO_PREPARE_SWITCH_FAILED, WhiteboxDefaultErrorEmit); @@ -513,10 +617,19 @@ static void RecoveryZone(UndoZone *uzone, uzone->SetLSN(uspMetaInfo->lsn); uzone->SetInsertURecPtr(uspMetaInfo->insertURecPtr); uzone->SetDiscardURecPtr(uspMetaInfo->discardURecPtr); + uzone->set_discard_urec_ptr_exrto(uspMetaInfo->discardURecPtr); uzone->SetForceDiscardURecPtr(uspMetaInfo->forceDiscardURecPtr); + uzone->set_force_discard_urec_ptr_exrto(uspMetaInfo->forceDiscardURecPtr); uzone->SetAllocateTSlotPtr(uspMetaInfo->allocateTSlotPtr); uzone->SetRecycleTSlotPtr(uspMetaInfo->recycleTSlotPtr); + uzone->set_recycle_tslot_ptr_exrto(uspMetaInfo->recycleTSlotPtr); uzone->SetRecycleXid(uspMetaInfo->recycleXid); + uzone->set_recycle_xid_exrto(uspMetaInfo->recycleXid); + ereport(DEBUG1, (errmodule(MOD_UNDO), errmsg(UNDOFORMAT("recovery_zone id:%d, lsn:%lu, " + "insert_urec_ptr:%lu, discard_urec_ptr:%lu, force_discard_urec_ptr:%lu, allocate_tslot_ptr:%lu, " + "recycle_tslot_ptr:%lu, recycle_xid:%lu."), zoneId, uspMetaInfo->lsn, uspMetaInfo->insertURecPtr, + uspMetaInfo->discardURecPtr, uspMetaInfo->forceDiscardURecPtr, uspMetaInfo->allocateTSlotPtr, + uspMetaInfo->recycleTSlotPtr, uspMetaInfo->recycleXid))); } /* Initialize parameters in the undo zone. */ @@ -528,11 +641,15 @@ void InitZone(UndoZone *uzone, const int zoneId, UndoPersistence upersistence) uzone->SetLSN(0); uzone->SetInsertURecPtr(UNDO_LOG_BLOCK_HEADER_SIZE); uzone->SetDiscardURecPtr(UNDO_LOG_BLOCK_HEADER_SIZE); + uzone->set_discard_urec_ptr_exrto(UNDO_LOG_BLOCK_HEADER_SIZE); uzone->SetForceDiscardURecPtr(UNDO_LOG_BLOCK_HEADER_SIZE); + uzone->set_force_discard_urec_ptr_exrto(UNDO_LOG_BLOCK_HEADER_SIZE); uzone->SetAllocateTSlotPtr(UNDO_LOG_BLOCK_HEADER_SIZE); uzone->SetRecycleTSlotPtr(UNDO_LOG_BLOCK_HEADER_SIZE); + uzone->set_recycle_tslot_ptr_exrto(UNDO_LOG_BLOCK_HEADER_SIZE); uzone->SetFrozenSlotPtr(INVALID_UNDO_SLOT_PTR); uzone->SetRecycleXid(InvalidTransactionId); + uzone->set_recycle_xid_exrto(InvalidTransactionId); uzone->SetFrozenXid(InvalidTransactionId); uzone->SetAttachPid(0); } @@ -544,6 +661,7 @@ void InitUndoSpace(UndoZone *uzone, UndoSpaceType type) usp->MarkClean(); usp->SetLSN(0); usp->SetHead(0); + usp->set_head_exrto(0); usp->SetTail(0); } diff --git a/src/gausskernel/storage/buffer/bufmgr.cpp b/src/gausskernel/storage/buffer/bufmgr.cpp index 311a80eb8..63cf83f6f 100644 --- a/src/gausskernel/storage/buffer/bufmgr.cpp +++ b/src/gausskernel/storage/buffer/bufmgr.cpp @@ -129,8 +129,7 @@ static int ts_ckpt_progress_comparator(Datum a, Datum b, void *arg); static bool ReadBuffer_common_ReadBlock(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode, bool isExtend, Block bufBlock, const XLogPhyBlock *pblk, bool *need_repair); -static Buffer ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, BlockNumber blockNum, - ReadBufferMode mode, BufferAccessStrategy strategy, bool *hit, const XLogPhyBlock *pblk); + static void TerminateBufferIO_common(BufferDesc *buf, bool clear_dirty, uint32 set_flag_bits); /* @@ -351,8 +350,6 @@ void ForgetPrivateRefCountEntry(PrivateRefCountEntry *ref) static void BufferSync(int flags); static void TerminateBufferIO_common(BufferDesc* buf, bool clear_dirty, uint32 set_flag_bits); void shared_buffer_write_error_callback(void* arg); -static BufferDesc* BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, BlockNumber blockNum, - BufferAccessStrategy strategy, bool* foundPtr, const XLogPhyBlock *pblk); static int rnode_comparator(const void* p1, const void* p2); @@ -1676,6 +1673,10 @@ Buffer ReadBuffer(Relation reln, BlockNumber block_num) Buffer ReadBufferExtended(Relation reln, ForkNumber fork_num, BlockNumber block_num, ReadBufferMode mode, BufferAccessStrategy strategy) { + if (IsExtremeRtoRunning() && !AmPageRedoWorker()) { + return standby_read_buf(reln, fork_num, block_num, mode, strategy); + } + bool hit = false; Buffer buf; @@ -2227,7 +2228,7 @@ static inline void BufferDescSetPBLK(BufferDesc *buf, const XLogPhyBlock *pblk) * * *hit is set to true if the request was satisfied from shared buffer cache. */ -static Buffer ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, BlockNumber blockNum, +Buffer ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode, BufferAccessStrategy strategy, bool *hit, const XLogPhyBlock *pblk) { BufferDesc *bufHdr = NULL; @@ -2288,7 +2289,7 @@ static Buffer ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumb * lookup the buffer. IO_IN_PROGRESS is set if the requested block is * not currently in memory. */ - bufHdr = BufferAlloc(smgr, relpersistence, forkNum, blockNum, strategy, &found, pblk); + bufHdr = BufferAlloc(smgr->smgr_rnode.node, relpersistence, forkNum, blockNum, strategy, &found, pblk); if (g_instance.attr.attr_security.enable_tde && IS_PGXC_DATANODE) { bufHdr->extra->encrypt = smgr->encrypt ? true : false; /* set tde flag */ } @@ -2670,14 +2671,15 @@ void PageCheckWhenChosedElimination(const BufferDesc *buf, uint32 oldFlags) * * No locks are held either at entry or exit. */ -static BufferDesc *BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber fork_num, BlockNumber block_num, - BufferAccessStrategy strategy, bool *found, const XLogPhyBlock *pblk) +BufferDesc *BufferAlloc(const RelFileNode &rel_file_node, char relpersistence, ForkNumber fork_num, + BlockNumber block_num, BufferAccessStrategy strategy, bool *found, + const XLogPhyBlock *pblk) { if (g_instance.attr.attr_storage.nvm_attr.enable_nvm) { - return NvmBufferAlloc(smgr, relpersistence, fork_num, block_num, strategy, found, pblk); + return NvmBufferAlloc(rel_file_node, relpersistence, fork_num, block_num, strategy, found, pblk); } - Assert(!IsSegmentPhysicalRelNode(smgr->smgr_rnode.node)); + Assert(!IsSegmentPhysicalRelNode(rel_file_node)); BufferTag new_tag; /* identity of requested block */ uint32 new_hash; /* hash value for newTag */ @@ -2692,7 +2694,7 @@ static BufferDesc *BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumbe uint32 buf_state; /* create a tag so we can lookup the buffer */ - INIT_BUFFERTAG(new_tag, smgr->smgr_rnode.node, fork_num, block_num); + INIT_BUFFERTAG(new_tag, rel_file_node, fork_num, block_num); /* determine its hash code and partition lock ID */ new_hash = BufTableHashCode(&new_tag); @@ -2844,8 +2846,8 @@ static BufferDesc *BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumbe } /* OK, do the I/O */ - TRACE_POSTGRESQL_BUFFER_WRITE_DIRTY_START(fork_num, block_num, smgr->smgr_rnode.node.spcNode, - smgr->smgr_rnode.node.dbNode, smgr->smgr_rnode.node.relNode); + TRACE_POSTGRESQL_BUFFER_WRITE_DIRTY_START(fork_num, block_num, rel_file_node.spcNode, + rel_file_node.dbNode, rel_file_node.relNode); /* during initdb, not need flush dw file */ if (dw_enabled() && pg_atomic_read_u32(&g_instance.ckpt_cxt_ctl->current_page_writer_count) > 0) { @@ -2868,8 +2870,8 @@ static BufferDesc *BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumbe ScheduleBufferTagForWriteback(t_thrd.storage_cxt.BackendWritebackContext, &buf->tag); - TRACE_POSTGRESQL_BUFFER_WRITE_DIRTY_DONE(fork_num, block_num, smgr->smgr_rnode.node.spcNode, - smgr->smgr_rnode.node.dbNode, smgr->smgr_rnode.node.relNode); + TRACE_POSTGRESQL_BUFFER_WRITE_DIRTY_DONE(fork_num, block_num, rel_file_node.spcNode, + rel_file_node.dbNode, rel_file_node.relNode); } else { /* * Someone else has locked the buffer, so give it up and loop @@ -5422,6 +5424,31 @@ void DropDatabaseBuffers(Oid dbid) gstrace_exit(GS_TRC_ID_DropDatabaseBuffers); } +void buffer_drop_exrto_standby_read_buffers() +{ + int i = 0; + ereport(LOG, (errmsg("buffer_drop_exrto_standby_read_buffers: start to drop buffers."))); + while (i < TOTAL_BUFFER_NUM) { + BufferDesc *buf_desc = GetBufferDescriptor(i); + uint32 buf_state; + /* + * Some safe unlocked checks can be done to reduce the number of cycle. + */ + if (!IS_EXRTO_RELFILENODE(buf_desc->tag.rnode)) { + i++; + continue; + } + + buf_state = LockBufHdr(buf_desc); + if (IS_EXRTO_RELFILENODE(buf_desc->tag.rnode)) { + InvalidateBuffer(buf_desc); /* with buffer head lock released */ + } else { + UnlockBufHdr(buf_desc, buf_state); + } + i++; + } +} + /* ----------------------------------------------------------------- * PrintBufferDescs * @@ -5690,6 +5717,12 @@ void MarkBufferDirtyHint(Buffer buffer, bool buffer_std) buf_desc = GetBufferDescriptor(buffer - 1); Assert(GetPrivateRefCount(buffer) > 0); + + // temp buf just for old page version, could not write to disk + if (pg_atomic_read_u32(&buf_desc->state) & BM_IS_TMP_BUF) { + return; + } + /* here, either share or exclusive lock is OK */ if (!LWLockHeldByMe(buf_desc->content_lock)) ereport(PANIC, (errcode(ERRCODE_INVALID_BUFFER), @@ -5723,8 +5756,9 @@ void MarkBufferDirtyHint(Buffer buffer, bool buffer_std) * The incremental checkpoint is protected by the doublewriter, the * half-write problem does not occur. */ - if (!ENABLE_INCRE_CKPT && XLogHintBitIsNeeded() && - (pg_atomic_read_u32(&buf_desc->state) & BM_PERMANENT)) { + bool need_write_wal = + (!ENABLE_INCRE_CKPT && XLogHintBitIsNeeded() && (pg_atomic_read_u32(&buf_desc->state) & BM_PERMANENT)); + if (need_write_wal) { /* * If we're in recovery we cannot dirty a page because of a hint. * We can set the hint, just not dirty the page as a result so the @@ -7352,3 +7386,18 @@ bool IsPageHitBufferPool(RelFileNode& node, ForkNumber forkNum, BlockNumber bloc } return false; } + +void buffer_in_progress_pop() +{ + Assert(t_thrd.storage_cxt.ParentInProgressBuf == NULL); + t_thrd.storage_cxt.ParentInProgressBuf = t_thrd.storage_cxt.InProgressBuf; + t_thrd.storage_cxt.ParentIsForInput = t_thrd.storage_cxt.IsForInput; + t_thrd.storage_cxt.InProgressBuf = NULL; +} + +void buffer_in_progress_push() +{ + t_thrd.storage_cxt.InProgressBuf = t_thrd.storage_cxt.ParentInProgressBuf; + t_thrd.storage_cxt.IsForInput = t_thrd.storage_cxt.ParentIsForInput; + t_thrd.storage_cxt.ParentInProgressBuf = NULL; +} diff --git a/src/gausskernel/storage/ipc/procarray.cpp b/src/gausskernel/storage/ipc/procarray.cpp index a2bec4d0e..b23a6b590 100755 --- a/src/gausskernel/storage/ipc/procarray.cpp +++ b/src/gausskernel/storage/ipc/procarray.cpp @@ -76,6 +76,7 @@ #include "access/clog.h" #include "access/csnlog.h" +#include "access/extreme_rto/page_redo.h" #include "access/subtrans.h" #include "access/transam.h" #include "access/twophase.h" @@ -541,6 +542,8 @@ void ProcArrayEndTransaction(PGPROC* proc, TransactionId latestXid, bool isCommi pgxact->xmin = InvalidTransactionId; proc->snapXmax = InvalidTransactionId; proc->snapCSN = InvalidCommitSeqNo; + proc->exrto_read_lsn = 0; + proc->exrto_gen_snap_time = 0; pgxact->csn_min = InvalidCommitSeqNo; pgxact->csn_dr = InvalidCommitSeqNo; /* must be cleared with xid/xmin: */ @@ -585,6 +588,8 @@ static inline void ProcArrayEndTransactionInternal(PGPROC* proc, PGXACT* pgxact, pgxact->xmin = InvalidTransactionId; proc->snapXmax = InvalidTransactionId; proc->snapCSN = InvalidCommitSeqNo; + proc->exrto_read_lsn = 0; + proc->exrto_gen_snap_time = 0; pgxact->csn_min = InvalidCommitSeqNo; pgxact->csn_dr = InvalidCommitSeqNo; /* must be cleared with xid/xmin: */ @@ -827,6 +832,8 @@ void ProcArrayClearTransaction(PGPROC* proc) /* Clear the subtransaction-XID cache too */ pgxact->nxids = 0; + proc->exrto_read_lsn = 0; + proc->exrto_gen_snap_time = 0; /* Free xid cache memory if needed */ ResetProcXidCache(proc, true); } @@ -2107,7 +2114,7 @@ Snapshot GetSnapshotData(Snapshot snapshot, bool force_local_snapshot) /* reset xmin before acquiring lwlock, in case blocking redo */ t_thrd.pgxact->xmin = InvalidTransactionId; RETRY_GET: - if (snapshot->takenDuringRecovery && !StreamThreadAmI() && + if (snapshot->takenDuringRecovery && !StreamThreadAmI() && !IS_EXRTO_READ && !u_sess->proc_cxt.clientIsCMAgent) { if (InterruptPending) { (void)pgstat_report_waitstatus(oldStatus); @@ -2429,6 +2436,10 @@ Snapshot GetSnapshotData(Snapshot snapshot, bool force_local_snapshot) (void)pgstat_report_waitstatus(oldStatus); } + if (IsExtremeRtoRunning() && pmState == PM_HOT_STANDBY) { + extreme_rto::exrto_read_snapshot(snapshot); + } + return snapshot; } @@ -3200,6 +3211,59 @@ ThreadId CancelVirtualTransaction(const VirtualTransactionId& vxid, ProcSignalRe return pid; } +bool proc_array_cancel_conflicting_proc(TransactionId latest_removed_xid, bool reach_max_check_times) +{ + ProcArrayStruct* proc_array = g_instance.proc_array_idx; + bool conflict = false; + + LWLockAcquire(ProcArrayLock, LW_SHARED); + for (int index = 0; index < proc_array->numProcs; index++) { + int pg_proc_no = proc_array->pgprocnos[index]; + PGPROC* pg_proc = g_instance.proc_base_all_procs[pg_proc_no]; + PGXACT* pg_xact = &g_instance.proc_base_all_xacts[pg_proc_no]; + XLogRecPtr read_lsn = pg_proc->exrto_read_lsn; + TransactionId pxmin = pg_xact->xmin; + + if (pg_proc->pid == 0 || !TransactionIdIsValid(pxmin) || XLogRecPtrIsInvalid(read_lsn)) { + continue; + } + + Assert(!(pg_xact->vacuumFlags & PROC_IN_VACUUM)); + /* + * Backend is doing logical decoding which manages xmin + * separately, check below. + */ + if (pg_xact->vacuumFlags & PROC_IN_LOGICAL_DECODING) { + continue; + } + + /* cancel query when its xmin < latest_removed_xid */ + if (TransactionIdPrecedesOrEquals(pxmin, latest_removed_xid)) { + conflict = true; + pg_proc->recoveryConflictPending = true; + if (pg_proc->pid != 0) { + /* + * Kill the pid if it's still here. If not, that's what we + * wanted so ignore any errors. + */ + (void)SendProcSignal(pg_proc->pid, PROCSIG_RECOVERY_CONFLICT_SNAPSHOT, pg_proc->backendId); + /* + * Wait a little bit for it to die so that we avoid flooding + * an unresponsive backend when system is heavily loaded. + */ + pg_usleep(5000L); + } + } + if (reach_max_check_times) { + ereport(WARNING, ( + errmsg("can not cancel thread while redo truncate, thread id = %lu", pg_proc->pid))); + } + } + LWLockRelease(ProcArrayLock); + + return conflict; +} + /* * MinimumActiveBackends --- count backends (other than myself) that are * in active transactions. Return true if the count exceeds the diff --git a/src/gausskernel/storage/lmgr/lwlocknames.txt b/src/gausskernel/storage/lmgr/lwlocknames.txt index dffbc1a6e..14a519173 100755 --- a/src/gausskernel/storage/lmgr/lwlocknames.txt +++ b/src/gausskernel/storage/lmgr/lwlocknames.txt @@ -140,3 +140,4 @@ DropArchiveSlotLock 130 AboCacheLock 131 OndemandXLogMemAllocLock 132 OndemandXLogFileHandleLock 133 +ExrtoSnapshotLock 134 diff --git a/src/gausskernel/storage/lmgr/proc.cpp b/src/gausskernel/storage/lmgr/proc.cpp index 8830f5863..29ee4bf0b 100755 --- a/src/gausskernel/storage/lmgr/proc.cpp +++ b/src/gausskernel/storage/lmgr/proc.cpp @@ -950,6 +950,8 @@ void InitProcess(void) t_thrd.proc->snap_refcnt_bitmap = 0; #endif + t_thrd.proc->exrto_read_lsn = 0; + t_thrd.proc->exrto_gen_snap_time = 0; /* Check that group locking fields are in a proper initial state. */ Assert(t_thrd.proc->lockGroupLeader == NULL); Assert(dlist_is_empty(&t_thrd.proc->lockGroupMembers)); @@ -1109,6 +1111,8 @@ void InitAuxiliaryProcess(void) t_thrd.pgxact->xmin = InvalidTransactionId; t_thrd.proc->snapXmax = InvalidTransactionId; t_thrd.proc->snapCSN = InvalidCommitSeqNo; + t_thrd.proc->exrto_read_lsn = 0; + t_thrd.proc->exrto_gen_snap_time = 0; t_thrd.pgxact->csn_min = InvalidCommitSeqNo; t_thrd.pgxact->csn_dr = InvalidCommitSeqNo; t_thrd.proc->backendId = InvalidBackendId; diff --git a/src/gausskernel/storage/nvm/nvmbuffer.cpp b/src/gausskernel/storage/nvm/nvmbuffer.cpp index 5ade48cab..0b3d5918c 100644 --- a/src/gausskernel/storage/nvm/nvmbuffer.cpp +++ b/src/gausskernel/storage/nvm/nvmbuffer.cpp @@ -255,10 +255,10 @@ static void NvmWaitBufferIO(BufferDesc *buf) return; } -BufferDesc *NvmBufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber fork_num, +BufferDesc *NvmBufferAlloc(const RelFileNode& rel_file_node, char relpersistence, ForkNumber fork_num, BlockNumber block_num, BufferAccessStrategy strategy, bool *found, const XLogPhyBlock *pblk) { - Assert(!IsSegmentPhysicalRelNode(smgr->smgr_rnode.node)); + Assert(!IsSegmentPhysicalRelNode(rel_file_node)); BufferTag new_tag; /* identity of requested block */ uint32 new_hash; /* hash value for newTag */ @@ -276,7 +276,7 @@ BufferDesc *NvmBufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber fo errno_t rc; /* create a tag so we can lookup the buffer */ - INIT_BUFFERTAG(new_tag, smgr->smgr_rnode.node, fork_num, block_num); + INIT_BUFFERTAG(new_tag, rel_file_node, fork_num, block_num); /* determine its hash code and partition lock ID */ new_hash = BufTableHashCode(&new_tag); diff --git a/src/gausskernel/storage/page/bufpage.cpp b/src/gausskernel/storage/page/bufpage.cpp index 153ab6bee..4b1a456c2 100644 --- a/src/gausskernel/storage/page/bufpage.cpp +++ b/src/gausskernel/storage/page/bufpage.cpp @@ -60,6 +60,7 @@ bool PageIsVerified(Page page, BlockNumber blkno) bool header_sane = false; bool all_zeroes = false; uint16 checksum = 0; + bool is_exrto_page = bool(p->pd_flags & PD_EXRTO_PAGE); /* * Don't verify page data unless the page passes basic non-zero test @@ -76,8 +77,8 @@ bool PageIsVerified(Page page, BlockNumber blkno) * the block can still reveal problems, which is why we offer the * checksum option. */ - if ((p->pd_flags & ~PD_VALID_FLAG_BITS) == 0 && p->pd_lower <= p->pd_upper && p->pd_upper <= p->pd_special && - p->pd_special <= BLCKSZ && p->pd_special == MAXALIGN(p->pd_special)) { + if (is_exrto_page || ((p->pd_flags & ~PD_VALID_FLAG_BITS) == 0 && p->pd_lower <= p->pd_upper && + p->pd_upper <= p->pd_special && p->pd_special <= BLCKSZ && p->pd_special == MAXALIGN(p->pd_special))) { header_sane = true; } diff --git a/src/gausskernel/storage/replication/basebackup.cpp b/src/gausskernel/storage/replication/basebackup.cpp index 9542a425e..e8523cfc6 100755 --- a/src/gausskernel/storage/replication/basebackup.cpp +++ b/src/gausskernel/storage/replication/basebackup.cpp @@ -19,6 +19,7 @@ #include "access/xlog_internal.h" /* for pg_start/stop_backup */ #include "access/cbmparsexlog.h" +#include "access/extreme_rto/standby_read/standby_read_base.h" #include "catalog/catalog.h" #include "catalog/pg_type.h" #include "gs_thread.h" @@ -1216,6 +1217,9 @@ bool IsSkipDir(const char * dirName) /* Skip temporary files */ if (strncmp(dirName, PG_TEMP_FILE_PREFIX, strlen(PG_TEMP_FILE_PREFIX)) == 0) return true; + if (strncmp(dirName, EXRTO_FILE_DIR, strlen(EXRTO_FILE_DIR)) == 0) { + return true; + } /* * If there's a backup_label file, it belongs to a backup started by diff --git a/src/gausskernel/storage/replication/slot.cpp b/src/gausskernel/storage/replication/slot.cpp index c26f39ceb..587929995 100644 --- a/src/gausskernel/storage/replication/slot.cpp +++ b/src/gausskernel/storage/replication/slot.cpp @@ -691,6 +691,8 @@ void ReplicationSlotRelease(void) LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE); t_thrd.pgxact->xmin = InvalidTransactionId; t_thrd.pgxact->vacuumFlags &= ~PROC_IN_LOGICAL_DECODING; + t_thrd.proc->exrto_read_lsn = 0; + t_thrd.proc->exrto_gen_snap_time = 0; LWLockRelease(ProcArrayLock); } diff --git a/src/gausskernel/storage/replication/walreceiver.cpp b/src/gausskernel/storage/replication/walreceiver.cpp index 9a2102421..ba49b9ae1 100755 --- a/src/gausskernel/storage/replication/walreceiver.cpp +++ b/src/gausskernel/storage/replication/walreceiver.cpp @@ -1699,6 +1699,8 @@ static void XLogWalRcvSendHSFeedback(void) else xmin = InvalidTransactionId; t_thrd.pgxact->xmin = InvalidTransactionId; + t_thrd.proc->exrto_read_lsn = 0; + t_thrd.proc->exrto_gen_snap_time = 0; /* * Always send feedback message. */ diff --git a/src/gausskernel/storage/replication/walsender.cpp b/src/gausskernel/storage/replication/walsender.cpp index 6d4a2ed3e..f15b692f6 100755 --- a/src/gausskernel/storage/replication/walsender.cpp +++ b/src/gausskernel/storage/replication/walsender.cpp @@ -2945,6 +2945,8 @@ static void PhysicalReplicationSlotNewXmin(TransactionId feedbackXmin) SpinLockAcquire(&slot->mutex); t_thrd.pgxact->xmin = InvalidTransactionId; + t_thrd.proc->exrto_read_lsn = 0; + t_thrd.proc->exrto_gen_snap_time = 0; /* * For physical replication we don't need the the interlock provided * by xmin and effective_xmin since the consequences of a missed increase diff --git a/src/gausskernel/storage/smgr/Makefile b/src/gausskernel/storage/smgr/Makefile index 0e7ef320b..b6bca4a44 100644 --- a/src/gausskernel/storage/smgr/Makefile +++ b/src/gausskernel/storage/smgr/Makefile @@ -9,7 +9,7 @@ ifneq "$(MAKECMDGOALS)" "clean" endif endif endif -OBJS = md.o smgr.o smgrtype.o knl_uundofile.o segstore.o page_compression.o +OBJS = md.o smgr.o smgrtype.o knl_uundofile.o segstore.o page_compression.o storage_exrto_file.o SUBDIRS = segment cfs diff --git a/src/gausskernel/storage/smgr/smgr.cpp b/src/gausskernel/storage/smgr/smgr.cpp index f4b4c60b3..b7ff7a4ff 100755 --- a/src/gausskernel/storage/smgr/smgr.cpp +++ b/src/gausskernel/storage/smgr/smgr.cpp @@ -122,14 +122,37 @@ static const f_smgr smgrsw[] = { seg_async_write, seg_move_buckets }, + + /* extreme-rto standby read */ + { + exrto_init, + NULL, + exrto_close, + NULL, + exrto_exists, + exrto_unlink, + exrto_extend, + NULL, + exrto_read, + exrto_write, + exrto_writeback, + exrto_nblocks, + exrto_truncate, + NULL, + NULL, + NULL, + NULL + } }; static const int NSmgr = lengthof(smgrsw); static void push_unlink_rel_one_fork_to_hashtbl(RelFileNode node, ForkNumber forkNum); -static inline int ChooseSmgrManager(RelFileNode rnode) +static inline int ChooseSmgrManager(const RelFileNode& rnode) { - if (rnode.dbNode == UNDO_DB_OID || rnode.dbNode == UNDO_SLOT_DB_OID) { + if (IS_EXRTO_RELFILENODE(rnode)) { + return EXRTO_MANAGER; + } else if (rnode.dbNode == UNDO_DB_OID || rnode.dbNode == UNDO_SLOT_DB_OID) { return UNDO_MANAGER; } else if (IsSegmentFileNode(rnode)) { return SEGMENT_MANAGER; @@ -313,7 +336,7 @@ SMgrRelation smgropen(const RelFileNode& rnode, BackendId backend, int col /* = reln->smgr_bcm_nblocks[colnum] = InvalidBlockNumber; } - if (reln->smgr_which == UNDO_MANAGER) { + if (reln->smgr_which == UNDO_MANAGER || reln->smgr_which == EXRTO_MANAGER) { fdNeeded = 1; } @@ -411,8 +434,15 @@ void smgrclose(SMgrRelation reln, BlockNumber blockNum) ereport(DEBUG5, (errmsg("smgr close %p", reln))); SMgrRelation* owner = NULL; int forknum; + int max_forknum; + + if (reln->smgr_which == EXRTO_MANAGER && reln->smgr_rnode.node.spcNode == EXRTO_BLOCK_INFO_SPACE_OID) { + max_forknum = EXRTO_FORK_NUM; + } else { + max_forknum = reln->md_fdarray_size; + } - for (forknum = 0; forknum < (int)(reln->md_fdarray_size); forknum++) { + for (forknum = 0; forknum < max_forknum; forknum++) { (*(smgrsw[reln->smgr_which].smgr_close))(reln, (ForkNumber)forknum, blockNum); } owner = reln->smgr_owner; @@ -567,12 +597,19 @@ void smgrdounlink(SMgrRelation reln, bool isRedo, BlockNumber blockNum) RelFileNodeBackend rnode = reln->smgr_rnode; int which = reln->smgr_which; int forknum; + int max_forknum; HTAB *unlink_rel_hashtbl = g_instance.bgwriter_cxt.unlink_rel_hashtbl; DelFileTag *entry = NULL; bool found = false; + if (which == EXRTO_MANAGER && reln->smgr_rnode.node.spcNode == EXRTO_BLOCK_INFO_SPACE_OID) { + max_forknum = EXRTO_FORK_NUM; + } else { + max_forknum = reln->md_fdarray_size; + } + /* Close the forks at smgr level */ - for (forknum = 0; forknum < (int)(reln->md_fdarray_size); forknum++) { + for (forknum = 0; forknum < max_forknum; forknum++) { (*(smgrsw[which].smgr_close))(reln, (ForkNumber)forknum, blockNum); } if (which == UNDO_MANAGER) { diff --git a/src/gausskernel/storage/smgr/storage_exrto_file.cpp b/src/gausskernel/storage/smgr/storage_exrto_file.cpp new file mode 100644 index 000000000..a3234cee4 --- /dev/null +++ b/src/gausskernel/storage/smgr/storage_exrto_file.cpp @@ -0,0 +1,545 @@ +/* + * Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * storage_exrto_file.cpp + * + * IDENTIFICATION + * src/gausskernel/storage/smgr/storage_exrto_file.cpp + * + * ------------------------------------------------------------------------- + */ + +#include "postgres.h" +#include "miscadmin.h" +#include "storage/smgr/fd.h" +#include "storage/vfd.h" +#include "storage/smgr/smgr.h" +#include "utils/memutils.h" +#include "access/extreme_rto/standby_read/standby_read_base.h" +#include "access/extreme_rto/standby_read/block_info_meta.h" + +const uint32 EXRTO_BASE_PAGE_FILE_BLOCKS = EXRTO_BASE_PAGE_FILE_MAXSIZE / BLCKSZ; +const uint32 EXRTO_LSN_INFO_FILE_BLOCKS = EXRTO_LSN_INFO_FILE_MAXSIZE / BLCKSZ; +const uint32 EXRTO_BLOCK_INFO_FILE_BLOCKS = RELSEG_SIZE; + +const int EXTEND_BLOCKS_NUM = 16; +const uint64 EXRTO_INVALID_BLOCK_NUMBER = 0xFFFFFFFFFFFFFFFFL; + +const uint32 EXRTO_FILE_SIZE[] = { + EXRTO_BASE_PAGE_FILE_MAXSIZE, EXRTO_LSN_INFO_FILE_MAXSIZE, EXRTO_BLOCK_INFO_FILE_MAXSIZE}; +const uint32 EXRTO_FILE_BLOCKS[] = { + EXRTO_BASE_PAGE_FILE_BLOCKS, EXRTO_LSN_INFO_FILE_BLOCKS, EXRTO_BLOCK_INFO_FILE_BLOCKS}; + +typedef struct _ExRTOFileState { + uint64 segno[EXRTO_FORK_NUM]; + File file[EXRTO_FORK_NUM]; +} ExRTOFileState; + +static inline ExRTOFileType exrto_file_type(uint32 space_oid) +{ + if (space_oid == EXRTO_BASE_PAGE_SPACE_OID) { + return BASE_PAGE; + } else if (space_oid == EXRTO_LSN_INFO_SPACE_OID) { + return LSN_INFO_META; + } else { + return BLOCK_INFO_META; + } +} + +static inline void set_file_state(ExRTOFileState *state, ForkNumber forknum, uint64 segno, File file) +{ + state->segno[forknum] = segno; + state->file[forknum] = file; +} + +static inline uint64 get_total_block_num(ExRTOFileType type, uint32 high, uint32 low) +{ + if (type == BASE_PAGE || type == LSN_INFO_META) { + return ((uint64)high << UINT64_HALF) | low; + } else { + return (uint64)low; + } +} + +static ExRTOFileState *alloc_file_state(void) +{ + MemoryContext current; + ExRTOFileState *state; + if (EnableLocalSysCache()) { + current = t_thrd.lsc_cxt.lsc->lsc_mydb_memcxt; + } else { + current = u_sess->storage_cxt.exrto_standby_read_file_cxt; + } + state = (ExRTOFileState *)MemoryContextAllocZero(current, sizeof(ExRTOFileState)); + for (int i = 0; i < EXRTO_FORK_NUM; i++) { + state->file[i] = -1; + } + + return state; +} + +static void exrto_get_file_path(const RelFileNode node, ForkNumber forknum, uint64 segno, char *path) +{ + ExRTOFileType type; + char filename[EXRTO_FILE_PATH_LEN]; + errno_t rc = EOK; + + type = exrto_file_type(node.spcNode); + if (type == BASE_PAGE || type == LSN_INFO_META) { + uint32 batch_id = node.dbNode >> LOW_WORKERID_BITS; + uint32 worker_id = node.dbNode & LOW_WORKERID_MASK; + rc = snprintf_s(filename, EXRTO_FILE_PATH_LEN, EXRTO_FILE_PATH_LEN - 1, "%02X%02X%016X", + batch_id, worker_id, segno); + } else { + rc = snprintf_s(filename, EXRTO_FILE_PATH_LEN, EXRTO_FILE_PATH_LEN - 1, "%u_%u_%s.%u", + node.dbNode, node.relNode, forkNames[forknum], (uint32)segno); + } + securec_check_ss(rc, "\0", "\0"); + + rc = snprintf_s(path, EXRTO_FILE_PATH_LEN, EXRTO_FILE_PATH_LEN - 1, "%s/%s/%s", + EXRTO_FILE_DIR, EXRTO_FILE_SUB_DIR[type], filename); + securec_check_ss(rc, "\0", "\0"); + + return; +} + +static uint64 get_seg_num(const RelFileNodeBackend& smgr_rnode, BlockNumber blocknum) +{ + ExRTOFileType type; + uint32 blocks_per_file; + uint64 total_blocknum; + uint64 segno; + + type = exrto_file_type(smgr_rnode.node.spcNode); + blocks_per_file = EXRTO_FILE_BLOCKS[type]; + total_blocknum = get_total_block_num(type, smgr_rnode.node.relNode, blocknum); + segno = (total_blocknum / blocks_per_file); + + return segno; +} + +static RelFileNodeForkNum exrto_file_relfilenode_forknum_fill(const RelFileNodeBackend &rnode, + ForkNumber forknum, uint64 segno) +{ + RelFileNodeForkNum node; + ExRTOFileType type; + + errno_t rc = memset_s(&node, sizeof(RelFileNodeForkNum), 0, sizeof(RelFileNodeForkNum)); + securec_check(rc, "", ""); + node.rnode = rnode; + type = exrto_file_type(rnode.node.spcNode); + if (type == BASE_PAGE || type == LSN_INFO_META) { + node.rnode.node.relNode = segno >> UINT64_HALF; + } + node.forknumber = forknum; + node.segno = (uint32)segno; + node.storage = ROW_STORE; + + return node; +} + +static ExRTOFileState *exrto_open_file(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, + ExtensionBehavior behavior) +{ + ExRTOFileState* state = (ExRTOFileState *)reln->fileState; + uint64 segno; + uint32 flags = O_RDWR | PG_BINARY; + char file_path[EXRTO_FILE_PATH_LEN]; + RelFileNodeForkNum filenode; + File fd; + + segno = get_seg_num(reln->smgr_rnode, blocknum); + /* No work if already open */ + if (state != NULL) { + if (state->file[forknum] > 0) { + if (state->segno[forknum] == segno) { + return state; + } + /* This is not the file we're looking for. */ + FileClose(state->file[forknum]); + } + } else { + state = alloc_file_state(); + reln->fileState = state; + } + set_file_state(state, forknum, 0, -1); + + if (behavior == EXTENSION_CREATE) { + flags |= O_CREAT; + } + ADIO_RUN() { + flags |= O_DIRECT; + } + ADIO_END(); + + exrto_get_file_path(reln->smgr_rnode.node, forknum, segno, file_path); + filenode = exrto_file_relfilenode_forknum_fill(reln->smgr_rnode, forknum, segno); + fd = DataFileIdOpenFile(file_path, filenode, (int)flags, S_IRUSR | S_IWUSR); + if (fd < 0) { + if ((behavior == EXTENSION_RETURN_NULL) && FILE_POSSIBLY_DELETED(errno)) { + return NULL; + } + exrto_close(reln, forknum, InvalidBlockNumber); + ereport(ERROR, + (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", file_path))); + } + + set_file_state(state, forknum, segno, fd); + + return state; +} + +BlockNumber get_single_file_nblocks(SMgrRelation reln, ForkNumber forknum, const ExRTOFileState*state) +{ + Assert(state != NULL); + + char *filename = FilePathName(state->file[forknum]); + off_t len = FileSeek(state->file[forknum], 0L, SEEK_END); + if (len < 0) { + exrto_close(reln, forknum, InvalidBlockNumber); + ereport(ERROR, (errcode_for_file_access(), + errmsg("could not seek to end of file \"%s\": %m", filename))); + } + + /* note that this calculation will ignore any partial block at EOF */ + return (BlockNumber)(len / BLCKSZ); +} + +void exrto_init(void) +{ + if (EnableLocalSysCache()) { + return; + } + Assert(u_sess->storage_cxt.exrto_standby_read_file_cxt == NULL); + u_sess->storage_cxt.exrto_standby_read_file_cxt = + AllocSetContextCreate(u_sess->top_mem_cxt, "ExrtoFileSmgr", ALLOCSET_DEFAULT_SIZES); +} + +void exrto_close(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum) +{ + ExRTOFileState* state = (ExRTOFileState*)reln->fileState; + + /* No work if already closed */ + if (state == NULL) { + return; + } + reln->fileState = NULL; /* prevent dangling pointer after error */ + + /* if not closed already */ + if (state->file[forknum] >= 0) { + FileClose(state->file[forknum]); + } + pfree(state); +} + +bool exrto_exists(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum) +{ + /* + * Close it first, to ensure that we notice if the fork has been unlinked + * since we opened it. + */ + exrto_close(reln, forknum, blocknum); + + bool isExist = false; + if (exrto_open_file(reln, forknum, blocknum, EXTENSION_RETURN_NULL) != NULL) { + isExist = true; + } + exrto_close(reln, forknum, blocknum); + return isExist; +} + +bool exrto_unlink_single_file(const RelFileNodeBackend &rnode, ForkNumber forknum, uint64 segno) +{ + struct stat stat_buf; + char segpath[EXRTO_FILE_PATH_LEN]; + + exrto_get_file_path(rnode.node, forknum, segno, segpath); + if (stat(segpath, &stat_buf) < 0) { + if (errno != ENOENT) { + ereport(WARNING, (errcode_for_file_access(), + errmsg("could not stat file \"%s\" before removing: %m", segpath))); + } + return false; + } + if (unlink(segpath) < 0) { + ereport(WARNING, (errcode_for_file_access(), + errmsg("could not remove file \"%s\": %m", segpath))); + } + return true; +} + +void exrto_unlink_file(const RelFileNodeBackend &rnode, ForkNumber forknum, BlockNumber blocknum) +{ + uint64 segno; + ExRTOFileType type = exrto_file_type(rnode.node.spcNode); + if (type == BLOCK_INFO_META) { + /* unlink all files */ + extreme_rto_standby_read::remove_block_meta_info_files_of_db(rnode.node.dbNode, rnode.node.relNode); + } else if (type == BASE_PAGE || type == LSN_INFO_META) { + /* just unlink the files before the file where blocknum is */ + segno = get_seg_num(rnode, blocknum); + while (segno != 0) { + segno -= 1; + if (!exrto_unlink_single_file(rnode, forknum, segno)) { + return; + } + } + } +} + +void exrto_unlink(const RelFileNodeBackend &rnode, ForkNumber forknum, bool is_redo, BlockNumber blocknum) +{ + ExRTOFileType type = exrto_file_type(rnode.node.spcNode); + if (type == BASE_PAGE || type == LSN_INFO_META) { + forknum = MAIN_FORKNUM; + } + if (forknum == InvalidForkNumber) { + for (int fork_num = 0; fork_num < EXRTO_FORK_NUM; fork_num++) { + exrto_unlink_file(rnode, (ForkNumber)fork_num, blocknum); + } + } else { + exrto_unlink_file(rnode, forknum, blocknum); + } +} + +/* extend EXTEND_BLOCKS_NUM pages */ +void exrto_extend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, char *buffer, bool skip_fsync) +{ + ExRTOFileState *state = NULL; + ExRTOFileType type; + uint64 total_block_num; + off_t seekpos; + int nbytes; + struct stat file_stat; + char* filename; + + type = exrto_file_type(reln->smgr_rnode.node.spcNode); + total_block_num = get_total_block_num(type, reln->smgr_rnode.node.relNode, blocknum); + if (total_block_num == EXRTO_INVALID_BLOCK_NUMBER) { + ereport(ERROR, + (errmsg("cannot extend file beyond %lu blocks.", EXRTO_INVALID_BLOCK_NUMBER))); + } + seekpos = (off_t)BLCKSZ * (total_block_num % EXRTO_FILE_BLOCKS[type]); + + state = exrto_open_file(reln, forknum, blocknum, EXTENSION_CREATE); + filename = FilePathName(state->file[forknum]); + if (stat(filename, &file_stat) < 0) { + exrto_close(reln, forknum, InvalidBlockNumber); + ereport(ERROR, (errmsg("could not stat file \"%s\": %m.", filename))); + } + Assert(file_stat.st_size % BLCKSZ == 0); + Assert(file_stat.st_size <= EXRTO_FILE_SIZE[type]); + + if (seekpos < file_stat.st_size) { + /* no need to extend */ + return; + } + + int extend_size = rtl::min(rtl::max(EXTEND_BLOCKS_NUM * BLCKSZ, (int)((seekpos - file_stat.st_size) + BLCKSZ)), + (int)(EXRTO_FILE_SIZE[type] - file_stat.st_size)); + nbytes = FilePWrite(state->file[forknum], NULL, extend_size, file_stat.st_size); + if (nbytes != extend_size) { + exrto_close(reln, forknum, InvalidBlockNumber); + if (nbytes < 0) { + ereport(ERROR, (errmsg("could not extend file \"%s\": %m.", filename))); + } + ereport(ERROR, + (errmsg("could not extend file \"%s\": wrote only %d of %d bytes.", filename, nbytes, extend_size))); + } + + Assert(get_single_file_nblocks(reln, forknum, state) <= ((BlockNumber)EXRTO_FILE_BLOCKS[type])); +} + +SMGR_READ_STATUS exrto_read(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, char *buffer) +{ + ExRTOFileState *state = NULL; + ExRTOFileType type; + ExtensionBehavior behavior; + uint64 total_block_num; + off_t seekpos; + int nbytes; + errno_t rc; + + type = exrto_file_type(reln->smgr_rnode.node.spcNode); + if (type == LSN_INFO_META || type == BLOCK_INFO_META) { + behavior = EXTENSION_RETURN_NULL; + } else { + behavior = EXTENSION_FAIL; + } + + total_block_num = get_total_block_num(type, reln->smgr_rnode.node.relNode, blocknum); + if (total_block_num == EXRTO_INVALID_BLOCK_NUMBER) { + ereport(ERROR, + (errmsg("cannot read file beyond %lu blocks.", EXRTO_INVALID_BLOCK_NUMBER))); + } + seekpos = (off_t)BLCKSZ * (total_block_num % EXRTO_FILE_BLOCKS[type]); + + state = exrto_open_file(reln, forknum, blocknum, behavior); + if (state == NULL) { + /* For lsn info and block info page, just set buffer to all zeros when not found on disk. */ + rc = memset_s(buffer, BLCKSZ, 0, BLCKSZ); + securec_check(rc, "\0", "\0"); + return SMGR_RD_OK; + } + + nbytes = FilePRead(state->file[forknum], buffer, BLCKSZ, seekpos); + if (nbytes == 0 && (type == LSN_INFO_META || type == BLOCK_INFO_META)) { + rc = memset_s(buffer, BLCKSZ, 0, BLCKSZ); + securec_check(rc, "\0", "\0"); + return SMGR_RD_OK; + } + if (nbytes != BLCKSZ) { + char *filename = FilePathName(state->file[forknum]); + exrto_close(reln, forknum, InvalidBlockNumber); + if (nbytes < 0) { + ereport(ERROR, + (errmsg("could not read block %u in file \"%s\": %m.", blocknum, filename))); + } + ereport(ERROR, + (errmsg("could not read block %u in file \"%s\": read only %d of %d bytes.", blocknum, filename, + nbytes, BLCKSZ))); + } + + if (PageIsVerified((Page)buffer, blocknum)) { + return SMGR_RD_OK; + } else { + return SMGR_RD_CRC_ERROR; + } +} + +void exrto_write(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, const char *buffer, bool skip_fsync) +{ + ExRTOFileState *state = NULL; + ExRTOFileType type; + uint64 total_block_num; + off_t seekpos; + int nbytes; + + type = exrto_file_type(reln->smgr_rnode.node.spcNode); + total_block_num = get_total_block_num(type, reln->smgr_rnode.node.relNode, blocknum); + if (total_block_num == EXRTO_INVALID_BLOCK_NUMBER) { + ereport(ERROR, + (errmsg("cannot write file beyond %lu blocks.", EXRTO_INVALID_BLOCK_NUMBER))); + } + seekpos = (off_t)BLCKSZ * (total_block_num % EXRTO_FILE_BLOCKS[type]); + + Assert(seekpos < (off_t)EXRTO_FILE_SIZE[type]); + + state = exrto_open_file(reln, forknum, blocknum, EXTENSION_CREATE); + nbytes = FilePWrite(state->file[forknum], buffer, BLCKSZ, seekpos); + if (nbytes != BLCKSZ) { + char *filename = FilePathName(state->file[forknum]); + exrto_close(reln, forknum, InvalidBlockNumber); + if (nbytes < 0) { + ereport(ERROR, + (errmsg("could not write block %u in file \"%s\": %m.", blocknum, filename))); + } + ereport(ERROR, + (errmsg("could not write block %u in file \"%s\": wrote only %d of %d bytes.", + blocknum, filename, nbytes, BLCKSZ))); + } +} + +BlockNumber exrto_nblocks(SMgrRelation, ForkNumber) +{ + return MaxBlockNumber; +} + +void exrto_truncate(SMgrRelation reln, ForkNumber forknum, BlockNumber nblocks) +{ + ExRTOFileType type = exrto_file_type(reln->smgr_rnode.node.spcNode); + Assert(type == BLOCK_INFO_META); + + BlockNumber curnblk = exrto_nblocks(reln, forknum); + if (curnblk == 0) { + return; + } + + if (nblocks > curnblk) { + ereport(ERROR, + (errcode_for_file_access(), errmsg("could not truncate file \"%s\" to %u blocks: it's only %u blocks now", + relpath(reln->smgr_rnode, forknum), nblocks, curnblk))); + } + if (nblocks == curnblk) { + return; + } + + uint32 blocks_per_file = EXRTO_FILE_BLOCKS[type]; + for (BlockNumber prior_blocks = 0;; prior_blocks += blocks_per_file) { + struct stat stat_buf; + char segpath[EXRTO_FILE_PATH_LEN]; + uint64 segno = get_seg_num(reln->smgr_rnode, prior_blocks); + exrto_get_file_path(reln->smgr_rnode.node, forknum, segno, segpath); + if (stat(segpath, &stat_buf) < 0) { + if (errno != ENOENT) { + ereport( + WARNING, + (errcode_for_file_access(), errmsg("could not stat file \"%s\" before truncate: %m", segpath))); + } + break; + } + + ExRTOFileState *state = exrto_open_file(reln, forknum, prior_blocks, EXTENSION_FAIL); + if (prior_blocks > nblocks) { + if (FileTruncate(state->file[forknum], 0) < 0) { + ereport(DEBUG1, + (errcode_for_file_access(), errmsg("could not truncate file \"%s\": %m", segpath))); + } + } else if (prior_blocks + ((BlockNumber)blocks_per_file) > nblocks) { + BlockNumber last_seg_block = nblocks - prior_blocks; + off_t truncate_offset = (off_t)last_seg_block * BLCKSZ; + + if (FileTruncate(state->file[forknum], truncate_offset) < 0) { + ereport(DEBUG1, + (errcode_for_file_access(), errmsg("could not truncate file \"%s\": %m", segpath))); + } + } + exrto_close(reln, forknum, InvalidBlockNumber); + } +} + +void exrto_writeback(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, BlockNumber nblocks) +{ + ExRTOFileType type; + uint64 total_block_num; + type = exrto_file_type(reln->smgr_rnode.node.spcNode); + total_block_num = get_total_block_num(type, reln->smgr_rnode.node.relNode, blocknum); + + while (nblocks > 0) { + BlockNumber nflush = nblocks; + off_t seekpos; + ExRTOFileState *state = NULL; + uint64 segnum_start, segnum_end; + state = exrto_open_file(reln, forknum, blocknum, EXTENSION_CREATE); + segnum_start = total_block_num / EXRTO_FILE_BLOCKS[type]; + segnum_end = (total_block_num + nblocks - 1) / EXRTO_FILE_BLOCKS[type]; + + if (segnum_start != segnum_end) { + nflush = EXRTO_FILE_BLOCKS[type] - (uint32)(total_block_num % EXRTO_FILE_BLOCKS[type]); + } + + Assert(nflush >= 1); + Assert(nflush <= nblocks); + + seekpos = (off_t)BLCKSZ * (total_block_num % EXRTO_FILE_BLOCKS[type]); + FileWriteback(state->file[forknum], seekpos, (off_t)BLCKSZ * nflush); + + nblocks -= nflush; + /* ensure that the relnode is not changed */ + Assert(((total_block_num + nflush) >> UINT64_HALF) == (total_block_num >> UINT64_HALF)); + total_block_num += nflush; + blocknum = (BlockNumber)total_block_num; + } +} diff --git a/src/include/access/extreme_rto/batch_redo.h b/src/include/access/extreme_rto/batch_redo.h index 54d2a5be0..55b0c1f5d 100644 --- a/src/include/access/extreme_rto/batch_redo.h +++ b/src/include/access/extreme_rto/batch_redo.h @@ -63,6 +63,26 @@ typedef struct redoitemhashentry { int redoItemNum; } RedoItemHashEntry; +inline void PRXLogRecGetBlockTag(XLogRecParseState *recordBlockState, RelFileNode *rnode, BlockNumber *blknum, + ForkNumber *forknum) +{ + XLogBlockParse *blockparse = &(recordBlockState->blockparse); + + if (rnode != NULL) { + rnode->dbNode = blockparse->blockhead.dbNode; + rnode->relNode = blockparse->blockhead.relNode; + rnode->spcNode = blockparse->blockhead.spcNode; + rnode->bucketNode = blockparse->blockhead.bucketNode; + rnode->opt = blockparse->blockhead.opt; + } + if (blknum != NULL) { + *blknum = blockparse->blockhead.blkno; + } + if (forknum != NULL) { + *forknum = blockparse->blockhead.forknum; + } +} + extern void PRPrintRedoItemHashTab(HTAB *redoItemHash); extern HTAB *PRRedoItemHashInitialize(MemoryContext context); extern void PRTrackClearBlock(XLogRecParseState *recordBlockState, HTAB *redoItemHash); diff --git a/src/include/access/extreme_rto/dispatcher.h b/src/include/access/extreme_rto/dispatcher.h index 70b3a5b48..ed5e61058 100644 --- a/src/include/access/extreme_rto/dispatcher.h +++ b/src/include/access/extreme_rto/dispatcher.h @@ -165,6 +165,7 @@ typedef struct { volatile bool recoveryStop; volatile XLogRedoNumStatics xlogStatics[RM_NEXT_ID][MAX_XLOG_INFO_NUM]; RedoTimeCost *startupTimeCost; + ExrtoSnapshotData exrto_snapshot; } LogDispatcher; typedef struct { diff --git a/src/include/access/extreme_rto/page_redo.h b/src/include/access/extreme_rto/page_redo.h index 3ffa739e6..7d789f858 100644 --- a/src/include/access/extreme_rto/page_redo.h +++ b/src/include/access/extreme_rto/page_redo.h @@ -33,8 +33,10 @@ #include "nodes/pg_list.h" #include "storage/proc.h" +#include "access/extreme_rto/batch_redo.h" #include "access/extreme_rto/posix_semaphore.h" #include "access/extreme_rto/spsc_blocking_queue.h" +#include "access/extreme_rto/standby_read/standby_read_base.h" #include "access/xlogproc.h" #include "postmaster/pagerepair.h" @@ -185,6 +187,7 @@ struct PageRedoWorker { HTAB *badPageHashTbl; char page[BLCKSZ]; XLogBlockDataParse *curRedoBlockState; + StandbyReadMetaInfo standby_read_meta_info; }; @@ -240,6 +243,7 @@ void DispatchClosefdMarkToAllRedoWorker(); void DispatchCleanInvalidPageMarkToAllRedoWorker(RepairFileKey key); const char *RedoWokerRole2Str(RedoRole role); +uint32 GetWorkerId(const RedoItemTag *redo_item_tag, uint32 worker_count); /* block or file repair function */ @@ -253,6 +257,9 @@ void BatchClearRecoveryThreadHashTbl(Oid spcNode, Oid dbNode); void RecordBadBlockAndPushToRemote(XLogBlockDataParse *datadecode, PageErrorType error_type, XLogRecPtr old_lsn, XLogPhyBlock pblk); void SeqCheckRemoteReadAndRepairPage(); - +void exrto_generate_snapshot(XLogRecPtr trxn_lsn); +void exrto_read_snapshot(Snapshot snapshot); +XLogRecPtr exrto_calculate_recycle_position(bool force_recyle); +TransactionId exrto_calculate_recycle_xmin_for_undo(); } // namespace extreme_rto #endif diff --git a/src/include/access/extreme_rto/standby_read.h b/src/include/access/extreme_rto/standby_read.h new file mode 100644 index 000000000..d54e3cc42 --- /dev/null +++ b/src/include/access/extreme_rto/standby_read.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * --------------------------------------------------------------------------------------- + * + * standby_read.h + * + * IDENTIFICATION + * src/include/access/extreme_rto/standby_read.h + * + * --------------------------------------------------------------------------------------- + */ + +#ifndef EXTREME_RTO_STANDBY_READ_H +#define EXTREME_RTO_STANDBY_READ_H + +namespace extreme_rto { +void exrto_recycle_main(); +} /* namespace extreme_rto */ +#endif diff --git a/src/include/access/extreme_rto/standby_read/block_info_meta.h b/src/include/access/extreme_rto/standby_read/block_info_meta.h new file mode 100644 index 000000000..b1d9eb18a --- /dev/null +++ b/src/include/access/extreme_rto/standby_read/block_info_meta.h @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * --------------------------------------------------------------------------------------- + * + * block_info_meta.h + * + * + * + * IDENTIFICATION + * src/include/access/extreme_rto/standby_read/block_info_meta.h + * + * --------------------------------------------------------------------------------------- + */ + +#ifndef BLOCK_INFO_META_H +#define BLOCK_INFO_META_H + +#include "gs_thread.h" +#include "postgres.h" +#include "access/xlogdefs.h" +#include "access/extreme_rto/standby_read/lsn_info_double_list.h" +#include "access/extreme_rto/standby_read/standby_read_base.h" +#include "storage/buf/bufpage.h" +#include "storage/buf/buf_internals.h" + +namespace extreme_rto_standby_read { + +const static uint32 BLOCK_INFO_PAGE_HEAD_PAD_SIZE = 40; +const static uint32 BLOCK_INFO_PAGE_VERSION = 1; // currently the first version of extreme rto standby read + +typedef struct _BlockInfoPageHeader { + PageXLogRecPtr lsn; /* LSN: next byte after last byte of wal record for last change to this page */ + uint16 checksum; /* checksum */ + uint16 flags; + uint32 version; + uint64 total_block_num; // all blocks of this table, only update on the first page + uint8 pad[BLOCK_INFO_PAGE_HEAD_PAD_SIZE]; +} BlockInfoPageHeader; + +#define BLOCK_INFO_PAGE_VALID_FLAG 0x0400 + +typedef struct _BlockMetaInfo { + uint32 timeline; + uint32 record_num; + XLogRecPtr min_lsn; + XLogRecPtr max_lsn; + uint32 flags; + uint32 pad; + LsnInfoDoubleList lsn_info_list; + LsnInfoDoubleList base_page_info_list; +} BlockMetaInfo; + +#define BLOCK_INFO_NODE_VALID_FLAG (1 << 24) +#define BLOCK_INFO_NODE_UPDATE_FLAG (1 << 25) +#define BLOCK_INFO_NODE_REFCOUNT_MASK 0xFFFFF +#define IS_BLOCK_INFO_UPDATING(_flags) ((_flags & BLOCK_INFO_NODE_UPDATE_FLAG) == BLOCK_INFO_NODE_UPDATE_FLAG) + +const static uint32 BLOCK_INFO_HEAD_SIZE = 64; // do not modify +const static uint32 BLOCK_INFO_SIZE = 64; // do not modify + +static const uint32 BLOCK_INFO_NUM_PER_PAGE = (BLCKSZ - BLOCK_INFO_HEAD_SIZE) / BLOCK_INFO_SIZE; + +typedef enum { + STANDBY_READ_RECLYE_NONE, + STANDBY_READ_RECLYE_UPDATE, + STANDBY_READ_RECLYE_ALL, +} StandbyReadRecyleState; + +BlockMetaInfo* get_block_meta_info_by_relfilenode( + const BufferTag& buf_tag, BufferAccessStrategy strategy, ReadBufferMode mode, Buffer* buffer); +void insert_lsn_to_block_info( + StandbyReadMetaInfo* mete_info, const BufferTag& buf_tag, const Page base_page, XLogRecPtr next_lsn); +StandbyReadRecyleState recyle_block_info( + const BufferTag& buf_tag, LsnInfoPosition base_page_info_pos, XLogRecPtr next_base_page_lsn, XLogRecPtr recyle_lsn); +bool get_page_lsn_info(const BufferTag& buf_tag, BufferAccessStrategy strategy, XLogRecPtr read_lsn, + StandbyReadLsnInfoArray* lsn_info); +static inline bool is_block_info_page_valid(BlockInfoPageHeader* header) +{ + return ((header->flags & BLOCK_INFO_PAGE_VALID_FLAG) == BLOCK_INFO_PAGE_VALID_FLAG); +} + +static inline bool is_block_meta_info_valid(BlockMetaInfo* meta_info) +{ + return (((meta_info->flags & BLOCK_INFO_NODE_VALID_FLAG) == BLOCK_INFO_NODE_VALID_FLAG) && + meta_info->timeline == t_thrd.shemem_ptr_cxt.ControlFile->timeline); +} + +void remove_one_block_info_file(const RelFileNode rnode); + +void remove_block_meta_info_files_of_db(Oid db_oid, Oid rel_oid = InvalidOid); + +} // namespace extreme_rto_standby_read + +#endif \ No newline at end of file diff --git a/src/include/access/extreme_rto/standby_read/lsn_info_double_list.h b/src/include/access/extreme_rto/standby_read/lsn_info_double_list.h new file mode 100644 index 000000000..c3df271e4 --- /dev/null +++ b/src/include/access/extreme_rto/standby_read/lsn_info_double_list.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * --------------------------------------------------------------------------------------- + * + * lsn_info_double_list.h + * + * + * + * IDENTIFICATION + * src/include/access/extreme_rto/standby_read/lsn_info_double_list.h + * + * --------------------------------------------------------------------------------------- + */ + +#ifndef LSN_INFO_DOUBLE_LIST_H +#define LSN_INFO_DOUBLE_LIST_H + +#include "gs_thread.h" +#include "postgres.h" +#include "access/extreme_rto/standby_read/standby_read_base.h" + +namespace extreme_rto_standby_read { +typedef uint64 LsnInfoPosition; + +static const LsnInfoPosition LSN_INFO_LIST_HEAD = 0xFFFFFFFFFFFFFFFFL; + +#define INFO_POSITION_IS_VALID(p) ((p) != 0xFFFFFFFFFFFFFFFFL) +#define INFO_POSITION_IS_INVALID(p) ((p) == 0xFFFFFFFFFFFFFFFFL) +typedef struct _LsnInfoDoubleList { + LsnInfoPosition prev; // not pointer, is position in lsn info meta table + LsnInfoPosition next; // not pointer, is position in lsn info meta table +} LsnInfoDoubleList; + +void lsn_info_list_init(LsnInfoDoubleList* node); +void info_list_modify_old_tail(StandbyReadMetaInfo *meta_info, LsnInfoPosition old_tail_pos, + LsnInfoPosition insert_pos, XLogRecPtr current_page_lsn, XLogRecPtr next_lsn, bool is_lsn_info); +} // namespace extreme_rto_standby_read +#endif \ No newline at end of file diff --git a/src/include/access/extreme_rto/standby_read/lsn_info_meta.h b/src/include/access/extreme_rto/standby_read/lsn_info_meta.h new file mode 100644 index 000000000..7694bb984 --- /dev/null +++ b/src/include/access/extreme_rto/standby_read/lsn_info_meta.h @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * --------------------------------------------------------------------------------------- + * + * lsn_info_meta.h + * + * + * + * IDENTIFICATION + * src/include/access/extreme_rto/standby_read/lsn_info_meta.h + * + * --------------------------------------------------------------------------------------- + */ + +#ifndef LSN_INFO_META_H +#define LSN_INFO_META_H + +#include "gs_thread.h" +#include "postgres.h" +#include "storage/buf/bufpage.h" +#include "storage/buf/buf_internals.h" +#include "access/extreme_rto/standby_read/lsn_info_double_list.h" +#include "access/extreme_rto/standby_read/standby_read_base.h" + +namespace extreme_rto_standby_read { +const static uint32 BASE_PAGE_MAP_SIZE = 16; +const static uint32 LSN_INFO_PAGE_HEAD_PAD_SIZE = 32; +const static uint32 LSN_INFO_PAGE_VERSION = 1; /* currently the first version of extreme rto standby read */ +const static uint32 LSN_NUM_PER_NODE = 5; +const static uint32 BYTE_BITS = 8; + +typedef struct _LsnInfoPageHeader { + PageXLogRecPtr lsn; /* LSN: next byte after last byte of wal record for last change to this page */ + uint16 checksum; /* checksum */ + uint16 flags; + uint32 version; + uint8 base_page_map[BASE_PAGE_MAP_SIZE]; + uint8 pad[LSN_INFO_PAGE_HEAD_PAD_SIZE]; +} LsnInfoPageHeader; + +typedef struct _LsnInfoNode { + LsnInfoDoubleList lsn_list; + uint32 flags; + uint16 type; + uint16 used; + XLogRecPtr lsn[LSN_NUM_PER_NODE]; +} LsnInfoNode; + +typedef struct _BasePageInfoNode { + LsnInfoNode lsn_info_node; + LsnInfoDoubleList base_page_list; + XLogRecPtr cur_page_lsn; + RelFileNode relfilenode; + ForkNumber fork_num; + BlockNumber block_num; + XLogRecPtr next_base_page_lsn; + BasePagePosition base_page_position; +} BasePageInfoNode; + +typedef LsnInfoNode* LsnInfo; +typedef BasePageInfoNode* BasePageInfo; + +const static uint32 LSN_INFO_HEAD_SIZE = 64; // do not modify +const static uint32 LSN_INFO_NODE_SIZE = 64; // do not modify +const static uint32 BASE_PAGE_INFO_NODE_SIZE = 128; // do not modify + +#define LSN_INFO_NODE_VALID_FLAG (1 << 24) +#define LSN_INFO_NODE_UPDATE_FLAG (1 << 25) +#define LSN_INFO_PAGE_VALID_FLAG 0x0400 + +typedef enum { + LSN_INFO_TYPE_BASE_PAGE = 1, + LSN_INFO_TYPE_LSNS, +} LsnInfoType; + +static inline bool is_lsn_info_node_valid(uint32 flags) +{ + return ((flags & LSN_INFO_NODE_VALID_FLAG) == LSN_INFO_NODE_VALID_FLAG); +} + +static inline bool is_lsn_info_node_updating(uint32 flags) +{ + return ((flags & LSN_INFO_NODE_UPDATE_FLAG) == LSN_INFO_NODE_UPDATE_FLAG); +} + +static inline bool is_lsn_info_page_valid(LsnInfoPageHeader *header) +{ + return ((header->flags & LSN_INFO_PAGE_VALID_FLAG) == LSN_INFO_PAGE_VALID_FLAG); +} + +static inline bool is_base_page_type(uint16 type) +{ + return (type == LSN_INFO_TYPE_BASE_PAGE); +} + +static inline bool is_lsn_type(uint16 type) +{ + return (type == LSN_INFO_TYPE_LSNS); +} + +inline uint32 lsn_info_postion_to_offset(LsnInfoPosition position) +{ + return position % BLCKSZ; +} + +static inline uint32 bit_to_offset(uint32 which_bit) +{ + return which_bit * LSN_INFO_NODE_SIZE; +} + +Page get_lsn_info_page(uint32 batch_id, uint32 worker_id, LsnInfoPosition position, ReadBufferMode mode, + Buffer* buffer); +void read_lsn_info_before(uint64 start_position, XLogRecPtr *readed_array, XLogRecPtr end_lsn); +LsnInfoDoubleList* lsn_info_position_to_node_ptr(LsnInfoPosition pos); + +// block meta table's page lock is held +void insert_lsn_to_lsn_info(StandbyReadMetaInfo* mete_info, LsnInfoDoubleList* head, + XLogRecPtr next_lsn); + +// block meta table's page lock is held +void insert_base_page_to_lsn_info(StandbyReadMetaInfo* meta_info, LsnInfoDoubleList* lsn_head, + LsnInfoDoubleList* base_page_head, const BufferTag& buf_tag, const Page base_page, XLogRecPtr curent_page_lsn, + XLogRecPtr next_lsn); + +void get_lsn_info_for_read(const BufferTag& buf_tag, LsnInfoPosition latest_lsn_base_page_pos, + StandbyReadLsnInfoArray* lsn_info_list, XLogRecPtr read_lsn); + +Buffer buffer_read_base_page(uint32 batch_id, uint32 redo_id, BasePagePosition position, ReadBufferMode mode); +void generate_base_page(StandbyReadMetaInfo* meta_info, const Page src_page); +void read_base_page(const BufferTag& buf_tag, BasePagePosition position, BufferDesc* dest_buf_desc); +void recycle_base_page_file(uint32 batch_id, uint32 redo_id, BasePagePosition recycle_pos); + +void set_base_page_map_bit(Page page, uint32 base_page_loc); +bool is_base_page_map_bit_set(Page page, uint32 which_bit); +void recycle_one_lsn_info_list(const BufferTag& buf_tag, LsnInfoPosition page_info_pos, + XLogRecPtr recycle_lsn, LsnInfoPosition *min_page_info_pos, XLogRecPtr *min_lsn); +void standby_read_recyle_per_workers(StandbyReadMetaInfo *standby_read_meta_info, XLogRecPtr recycle_lsn); + +} // namespace extreme_rto_standby_read +#endif \ No newline at end of file diff --git a/src/include/access/extreme_rto/standby_read/standby_read_base.h b/src/include/access/extreme_rto/standby_read/standby_read_base.h new file mode 100644 index 000000000..714b47506 --- /dev/null +++ b/src/include/access/extreme_rto/standby_read/standby_read_base.h @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * --------------------------------------------------------------------------------------- + * + * standby_read_base.h + * + * + * + * IDENTIFICATION + * src/include/access/extreme_rto/standby_read/standby_read_base.h + * + * --------------------------------------------------------------------------------------- + */ + +#ifndef STANDBY_READ_BASE_H +#define STANDBY_READ_BASE_H + +#include "gs_thread.h" +#include "postgres.h" +#include "storage/buf/bufpage.h" +#include "postmaster/alarmchecker.h" + +#define EXRTO_FILE_DIR "standby_read" +#define EXRTO_OLD_FILE_DIR "standby_read_old" + +static const uint32 EXRTO_BASE_PAGE_FILE_MAXSIZE = 64 * 1024 * 1024; /* 64MB */ +static const uint32 EXRTO_LSN_INFO_FILE_MAXSIZE = 16 * 1024 * 1024; /* 16MB */ +static const uint32 EXRTO_BLOCK_INFO_FILE_MAXSIZE = RELSEG_SIZE * BLCKSZ; + +extern const char* EXRTO_FILE_SUB_DIR[]; +extern const uint32 EXRTO_FILE_PATH_LEN; + +#define UINT64_HALF 32 +#define LOW_WORKERID_BITS 16 +#define LOW_WORKERID_MASK ((1U << LOW_WORKERID_BITS) - 1) + +#define EXRTODEBUGINFO , __FUNCTION__, __LINE__ +#define EXRTODEBUGSTR "[%s:%d]" +#define EXRTOFORMAT(f) EXRTODEBUGSTR f EXRTODEBUGINFO + +enum ExRTOFileType { + BASE_PAGE = 0, + LSN_INFO_META, + BLOCK_INFO_META, +}; + +typedef uint64 BasePagePosition; + +typedef struct _StandbyReadMetaInfo { + uint32 batch_id; + uint32 redo_id; + uint64 lsn_table_recyle_position; + uint64 lsn_table_next_position; // next position can insert node, shoud jump page header before use + BasePagePosition base_page_recyle_position; + BasePagePosition base_page_next_position; // next position can insert page + XLogRecPtr recycle_lsn_per_worker; +} StandbyReadMetaInfo; + +inline void standby_read_meta_page_set_lsn(Page page, XLogRecPtr LSN) +{ + if (XLByteLT(LSN, PageGetLSN(page))) { + return; + } + PageSetLSNInternal(page, LSN); +} + +void exrto_clean_dir(void); +void exrto_recycle_old_dir(void); +void exrto_standby_read_init(); +#endif \ No newline at end of file diff --git a/src/include/access/multi_redo_api.h b/src/include/access/multi_redo_api.h index 750e5b451..331bcd897 100644 --- a/src/include/access/multi_redo_api.h +++ b/src/include/access/multi_redo_api.h @@ -34,6 +34,7 @@ #include "storage/proc.h" #include "access/redo_statistic.h" #include "access/extreme_rto_redo_api.h" +#include "postmaster/postmaster.h" #ifdef ENABLE_LITE_MODE #define ENABLE_ONDEMAND_RECOVERY false @@ -65,6 +66,9 @@ static const uint32 PAGE_REDO_WORKER_READY = 2; static const uint32 PAGE_REDO_WORKER_EXIT = 3; static const uint32 BIG_RECORD_LENGTH = XLOG_BLCKSZ * 16; +#define IS_EXRTO_READ (g_instance.attr.attr_storage.EnableHotStandby && IsExtremeRedo()) +#define IS_EXRTO_STANDBY_READ (IS_EXRTO_READ && pm_state_is_hot_standby()) + static inline int get_real_recovery_parallelism() { return g_instance.attr.attr_storage.real_recovery_parallelism; diff --git a/src/include/access/ustore/undo/knl_uundospace.h b/src/include/access/ustore/undo/knl_uundospace.h index e5cc112d7..cd9832e79 100644 --- a/src/include/access/ustore/undo/knl_uundospace.h +++ b/src/include/access/ustore/undo/knl_uundospace.h @@ -51,6 +51,10 @@ class UndoSpace { { return this->head_; } + inline UndoLogOffset Head_exrto(void) + { + return this->head_exrto; + } inline UndoLogOffset Tail(void) { return this->tail_; @@ -66,6 +70,10 @@ class UndoSpace { { this->head_ = head; } + inline void set_head_exrto(UndoRecPtr head) + { + this->head_exrto = head; + } inline void SetTail(UndoRecPtr tail) { this->tail_ = tail; @@ -109,10 +117,14 @@ class UndoSpace { void CreateNonExistsUndoFile(int zid, uint32 dbId); static void CheckPointUndoSpace(int fd, UndoSpaceType type); static void RecoveryUndoSpace(int fd, UndoSpaceType type); + UndoLogOffset find_oldest_offset(int zid, uint32 db_id) const; + void unlink_residual_log(int zid, UndoLogOffset start, UndoLogOffset end, uint32 db_id) const; private: /* next insertion point (head), this backend is the only one that can modify insert. */ UndoLogOffset head_; + /* real next insertion point (head), this backend is the only one that can modify insert. */ + UndoLogOffset head_exrto; /* one past end of highest segment, need lock befor modify end. */ UndoLogOffset tail_; diff --git a/src/include/access/ustore/undo/knl_uundozone.h b/src/include/access/ustore/undo/knl_uundozone.h index a2402299f..5d87696ae 100644 --- a/src/include/access/ustore/undo/knl_uundozone.h +++ b/src/include/access/ustore/undo/knl_uundozone.h @@ -119,6 +119,10 @@ class UndoZone : public BaseObject { { return MAKE_UNDO_PTR(zid_, recycleTSlotPtr_); } + inline UndoSlotPtr get_recycle_tslot_ptr_exrto(void) + { + return MAKE_UNDO_PTR(zid_, recycle_tslot_ptr_exrto); + } inline UndoSlotPtr GetFrozenSlotPtr(void) { return frozenSlotPtr_; @@ -127,6 +131,10 @@ class UndoZone : public BaseObject { { return recycleXid_; } + inline TransactionId get_recycle_xid_exrto(void) + { + return recycle_xid_exrto; + } inline TransactionId GetFrozenXid(void) { return frozenXid_; @@ -156,10 +164,18 @@ class UndoZone : public BaseObject { { discardURecPtr_ = UNDO_PTR_GET_OFFSET(discard); } + inline void set_discard_urec_ptr_exrto(UndoRecPtr discard) + { + discard_urec_ptr_exrto = UNDO_PTR_GET_OFFSET(discard); + } inline void SetForceDiscardURecPtr(UndoRecPtr discard) { forceDiscardURecPtr_ = UNDO_PTR_GET_OFFSET(discard); - } + } + inline void set_force_discard_urec_ptr_exrto(UndoRecPtr discard) + { + force_discard_urec_ptr_exrto = UNDO_PTR_GET_OFFSET(discard); + } inline void SetAttachPid(ThreadId attachPid) { attachPid_ = attachPid; @@ -176,6 +192,10 @@ class UndoZone : public BaseObject { { recycleTSlotPtr_ = UNDO_PTR_GET_OFFSET(recycle); } + inline void set_recycle_tslot_ptr_exrto(UndoSlotPtr recycle) + { + recycle_tslot_ptr_exrto = UNDO_PTR_GET_OFFSET(recycle); + } inline void SetLSN(XLogRecPtr lsn) { lsn_ = lsn; @@ -188,6 +208,10 @@ class UndoZone : public BaseObject { { recycleXid_ = recycleXid; } + inline void set_recycle_xid_exrto(TransactionId recycle_xid) + { + recycle_xid_exrto = recycle_xid; + } inline void SetFrozenXid(TransactionId frozenXid) { frozenXid_ = frozenXid; @@ -200,6 +224,10 @@ class UndoZone : public BaseObject { { return insertURecPtr_ != forceDiscardURecPtr_; } + inline bool Used_exrto(void) + { + return insertURecPtr_ != force_discard_urec_ptr_exrto; + } /* Lock and unlock undozone. */ void InitLock(void) { @@ -300,6 +328,10 @@ class UndoZone : public BaseObject { /* Recovery undospace info from persistent file. */ static void RecoveryUndoZone(int fd); + UndoRecordState check_record_valid_exrto(UndoLogOffset offset, bool check_force_recycle, + TransactionId *last_xid) const; + uint64 release_residual_record_space(); + uint64 release_residual_slot_space(); private: static const uint32 UNDO_ZONE_ATTACHED = 1; @@ -316,6 +348,13 @@ class UndoZone : public BaseObject { TransactionId recycleXid_; TransactionId frozenXid_; ThreadId attachPid_; + + /* for extreme RTO read. */ + UndoSlotOffset recycle_tslot_ptr_exrto; + UndoLogOffset discard_urec_ptr_exrto; + UndoLogOffset force_discard_urec_ptr_exrto; + TransactionId recycle_xid_exrto; + /* Need Lock undo zone before alloc, preventing from checkpoint. */ LWLock *lock_; /* Lsn for undo zone meta. */ diff --git a/src/include/access/xlogproc.h b/src/include/access/xlogproc.h index fba9b6e3f..ed9d926e7 100755 --- a/src/include/access/xlogproc.h +++ b/src/include/access/xlogproc.h @@ -216,6 +216,7 @@ typedef enum { typedef struct { uint32 blockddltype; int rels; + uint32 mainDataLen; char *mainData; bool compress; } XLogBlockDdlParse; @@ -947,6 +948,10 @@ static inline Buffer AtomicExchangeBuffer(volatile Buffer *ptr, Buffer newval) return old; } +/* this is an estimated value */ +static const uint32 MAX_BUFFER_NUM_PER_WAL_RECORD = XLR_MAX_BLOCK_ID + 1; +static const uint32 LSN_MOVE32 = 10; + void HeapXlogCleanOperatorPage( RedoBufferInfo* buffer, void* recorddata, void* blkdata, Size datalen, Size* freespace, bool repairFragmentation); void HeapXlogFreezeOperatorPage(RedoBufferInfo* buffer, void* recorddata, void* blkdata, Size datalen, @@ -1117,7 +1122,7 @@ void SegPageRedoDataBlock(XLogBlockHead *blockhead, XLogBlockDataParse *blockdat extern void xlog_redo_data_block( XLogBlockHead* blockhead, XLogBlockDataParse* blockdatarec, RedoBufferInfo* bufferinfo); extern void XLogRecSetBlockDdlState(XLogBlockDdlParse* blockddlstate, uint32 blockddltype, char *mainData, - int rels = 1, bool compress = false); + int rels = 1, bool compress = false, uint32 main_data_len = 0); XLogRedoAction XLogCheckBlockDataRedoAction(XLogBlockDataParse* datadecode, RedoBufferInfo* bufferinfo); void BtreeRedoDataBlock(XLogBlockHead* blockhead, XLogBlockDataParse* blockdatarec, RedoBufferInfo* bufferinfo); @@ -1275,5 +1280,6 @@ extern bool IsCheckPoint(const XLogRecParseState *parseState); void redo_atomic_xlog_dispatch(uint8 opCode, RedoBufferInfo *redo_buf, const char *data); void seg_redo_new_page_copy_and_flush(BufferTag *tag, char *data, XLogRecPtr lsn); +void redo_target_page(const BufferTag& buf_tag, StandbyReadLsnInfoArray* lsn_info, Buffer base_page_buf); #endif diff --git a/src/include/catalog/storage.h b/src/include/catalog/storage.h index 66dcbb71c..200f0c189 100644 --- a/src/include/catalog/storage.h +++ b/src/include/catalog/storage.h @@ -27,8 +27,8 @@ extern void RelationCreateStorage(RelFileNode rnode, char relpersistence, Oid ow Relation rel = NULL); extern void RelationDropStorage(Relation rel, bool isDfsTruncate = false); extern void RelationPreserveStorage(RelFileNode rnode, bool atCommit); -extern void RelationTruncate(Relation rel, BlockNumber nblocks); -extern void PartitionTruncate(Relation parent, Partition part, BlockNumber nblocks); +extern void RelationTruncate(Relation rel, BlockNumber nblocks, TransactionId latest_removed_xid = InvalidTransactionId); +extern void PartitionTruncate(Relation parent, Partition part, BlockNumber nblocks, TransactionId latest_removed_xid = InvalidTransactionId); extern void PartitionDropStorage(Relation rel, Partition part); extern void BucketCreateStorage(RelFileNode rnode, Oid bucketOid, Oid ownerid); extern void InsertStorageIntoPendingList(_in_ const RelFileNode* rnode, _in_ AttrNumber attrnum, _in_ BackendId backend, diff --git a/src/include/catalog/storage_xlog.h b/src/include/catalog/storage_xlog.h index 841c42fd5..1f3ce16b6 100644 --- a/src/include/catalog/storage_xlog.h +++ b/src/include/catalog/storage_xlog.h @@ -51,8 +51,11 @@ typedef struct xl_smgr_truncate { typedef struct xl_smgr_truncate_compress { xl_smgr_truncate xlrec; uint2 pageCompressOpts; + TransactionId latest_removed_xid; } xl_smgr_truncate_compress; +#define TRUNCATE_CONTAIN_XID_SIZE (offsetof(xl_smgr_truncate_compress, latest_removed_xid) + sizeof(TransactionId)) + extern void log_smgrcreate(RelFileNode *rnode, ForkNumber forkNum); extern void smgr_redo(XLogReaderState *record); @@ -60,7 +63,8 @@ extern void smgr_desc(StringInfo buf, XLogReaderState *record); extern const char* smgr_type_name(uint8 subtype); extern void smgr_redo_create(RelFileNode rnode, ForkNumber forkNum, char *data); -extern void xlog_block_smgr_redo_truncate(RelFileNode rnode, BlockNumber blkno, XLogRecPtr lsn); +extern void xlog_block_smgr_redo_truncate(RelFileNode rnode, BlockNumber blkno, XLogRecPtr lsn, + TransactionId latest_removed_xid); /* An xlog combined by multiply sub-xlog, it will be decoded again */ #define XLOG_SEG_ATOMIC_OPERATION 0x00 diff --git a/src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback-post_catalog_maindb_92_909.sql b/src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback-post_catalog_maindb_92_909.sql new file mode 100644 index 000000000..1ceaa4bdf --- /dev/null +++ b/src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback-post_catalog_maindb_92_909.sql @@ -0,0 +1 @@ +DROP FUNCTION IF EXISTS pg_catalog.gs_hot_standby_space_info() cascade; \ No newline at end of file diff --git a/src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback-post_catalog_otherdb_92_909.sql b/src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback-post_catalog_otherdb_92_909.sql new file mode 100644 index 000000000..1ceaa4bdf --- /dev/null +++ b/src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback-post_catalog_otherdb_92_909.sql @@ -0,0 +1 @@ +DROP FUNCTION IF EXISTS pg_catalog.gs_hot_standby_space_info() cascade; \ No newline at end of file diff --git a/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_909.sql b/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_909.sql new file mode 100644 index 000000000..9317bbf1f --- /dev/null +++ b/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_909.sql @@ -0,0 +1,11 @@ +DROP FUNCTION IF EXISTS pg_catalog.gs_hot_standby_space_info() cascade; +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 6218; +CREATE OR REPLACE FUNCTION pg_catalog.gs_hot_standby_space_info +( OUT base_page_file_num xid, + OUT base_page_total_size xid, + OUT lsn_info_meta_file_num xid, + OUT lsn_info_meta_total_size xid, + OUT block_info_meta_file_num xid, + OUT block_info_meta_total_size xid + ) +RETURNS SETOF record LANGUAGE INTERNAL ROWS 1 STRICT as 'gs_hot_standby_space_info'; \ No newline at end of file diff --git a/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_909.sql b/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_909.sql new file mode 100644 index 000000000..9317bbf1f --- /dev/null +++ b/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_909.sql @@ -0,0 +1,11 @@ +DROP FUNCTION IF EXISTS pg_catalog.gs_hot_standby_space_info() cascade; +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 6218; +CREATE OR REPLACE FUNCTION pg_catalog.gs_hot_standby_space_info +( OUT base_page_file_num xid, + OUT base_page_total_size xid, + OUT lsn_info_meta_file_num xid, + OUT lsn_info_meta_total_size xid, + OUT block_info_meta_file_num xid, + OUT block_info_meta_total_size xid + ) +RETURNS SETOF record LANGUAGE INTERNAL ROWS 1 STRICT as 'gs_hot_standby_space_info'; \ No newline at end of file diff --git a/src/include/gs_thread.h b/src/include/gs_thread.h index b8caca6ae..8427048be 100755 --- a/src/include/gs_thread.h +++ b/src/include/gs_thread.h @@ -124,6 +124,7 @@ typedef enum knl_thread_role { APPLY_WORKER, STACK_PERF_WORKER, DMS_AUXILIARY_THREAD, + EXRTO_RECYCLER, BARRIER_PREPARSE, TS_COMPACTION, TS_COMPACTION_CONSUMER, diff --git a/src/include/knl/knl_guc/knl_instance_attr_storage.h b/src/include/knl/knl_guc/knl_instance_attr_storage.h index 7161e8d11..b9c572b81 100755 --- a/src/include/knl/knl_guc/knl_instance_attr_storage.h +++ b/src/include/knl/knl_guc/knl_instance_attr_storage.h @@ -208,6 +208,14 @@ typedef struct knl_instance_attr_storage { int max_logical_replication_workers; char *redo_bind_cpu_attr; int max_active_gtt; + + /* extreme-rto standby read */ + int64 max_standby_base_page_size; + int64 max_standby_lsn_info_size; + int base_page_saved_interval; + double standby_force_recyle_ratio; + int standby_recycle_interval; + int standby_max_query_time; #ifndef ENABLE_MULTIPLE_NODES bool enable_save_confirmed_lsn; #endif diff --git a/src/include/knl/knl_instance.h b/src/include/knl/knl_instance.h index b5ae35361..592d4e033 100755 --- a/src/include/knl/knl_instance.h +++ b/src/include/knl/knl_instance.h @@ -113,6 +113,8 @@ enum knl_parallel_redo_state { REDO_DONE, }; +typedef struct ExrtoSnapshotData* ExrtoSnapshot; + /* all process level attribute which expose to user */ typedef struct knl_instance_attr { @@ -216,6 +218,7 @@ typedef struct knl_g_pid_context { ThreadId LogicalReadWorkerPID; ThreadId LogicalDecoderWorkerPID; ThreadId BarrierPreParsePID; + ThreadId exrto_recycler_pid; ThreadId ApplyLauncerPID; ThreadId StackPerfPID; ThreadId CfsShrinkerPID; @@ -746,7 +749,7 @@ typedef struct knl_g_parallel_redo_context { char* ali_buf; XLogRedoNumStatics xlogStatics[RM_NEXT_ID][MAX_XLOG_INFO_NUM]; RedoCpuBindControl redoCpuBindcontrl; - + XLogRecPtr global_recycle_lsn; /* extreme-rto standby read */ HTAB **redoItemHash; /* used in ondemand extreme RTO */ } knl_g_parallel_redo_context; @@ -918,6 +921,7 @@ typedef struct knl_g_undo_context { pg_atomic_uint64 globalFrozenXid; /* Oldest transaction id which is having undo. */ pg_atomic_uint64 globalRecycleXid; + bool is_exrto_residual_undo_file_recycled; } knl_g_undo_context; typedef struct knl_g_flashback_context { diff --git a/src/include/knl/knl_session.h b/src/include/knl/knl_session.h index 0112d3a7b..aaadd977a 100644 --- a/src/include/knl/knl_session.h +++ b/src/include/knl/knl_session.h @@ -685,6 +685,9 @@ typedef struct knl_u_utils_context { HTAB* set_user_params_htab; DestReceiver* spi_printtupDR; + + /* backend read lsn for read on standby in extreme rto */ + XLogRecPtr exrto_read_lsn; } knl_u_utils_context; typedef struct knl_u_security_context { @@ -1851,6 +1854,9 @@ typedef struct knl_u_storage_context { /* md.cpp */ MemoryContext MdCxt; /* context for all md.c allocations */ + /* exrto_file.cpp */ + MemoryContext exrto_standby_read_file_cxt; + /* sync.cpp */ MemoryContext pendingOpsCxt; struct HTAB *pendingOps; diff --git a/src/include/knl/knl_thread.h b/src/include/knl/knl_thread.h index 12d8a18f9..088c655bb 100755 --- a/src/include/knl/knl_thread.h +++ b/src/include/knl/knl_thread.h @@ -77,6 +77,7 @@ #include "port/pg_crc32c.h" #include "ddes/dms/ss_common_attr.h" #include "ddes/dms/ss_txnstatus.h" +#include "access/extreme_rto/standby_read/standby_read_base.h" #define MAX_PATH_LEN 1024 extern const int g_reserve_param_num; @@ -1944,8 +1945,22 @@ typedef struct { volatile sig_atomic_t got_SIGHUP; volatile sig_atomic_t sleep_long; volatile sig_atomic_t check_repair; + void *redo_worker_ptr; } knl_t_page_redo_context; +typedef struct _StandbyReadLsnInfoArray { + XLogRecPtr *lsn_array; + uint32 lsn_num; + XLogRecPtr base_page_lsn; + BasePagePosition base_page_pos; +} StandbyReadLsnInfoArray; + +typedef struct { + volatile sig_atomic_t shutdown_requested; + volatile sig_atomic_t got_SIGHUP; + StandbyReadLsnInfoArray lsn_info; +} knl_t_exrto_recycle_context; + typedef struct knl_t_startup_context { /* * Flags set by interrupt handlers for later service in the redo loop. @@ -2564,8 +2579,10 @@ typedef struct knl_t_storage_context { struct HTAB* SharedBufHash; struct HTAB* BufFreeListHash; struct BufferDesc* InProgressBuf; + struct BufferDesc* ParentInProgressBuf; /* local state for StartBufferIO and related functions */ volatile bool IsForInput; + volatile bool ParentIsForInput; /* local state for LockBufferForCleanup */ struct BufferDesc* PinCountWaitBuf; /* local state for aio clean up resource */ @@ -3483,6 +3500,7 @@ typedef struct knl_thrd_context { knl_t_percentile_context percentile_cxt; knl_t_perf_snap_context perf_snap_cxt; knl_t_page_redo_context page_redo_cxt; + knl_t_exrto_recycle_context exrto_recycle_cxt; knl_t_parallel_decode_worker_context parallel_decode_cxt; knl_t_logical_read_worker_context logicalreadworker_cxt; knl_t_heartbeat_context heartbeat_cxt; diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h index e55ceaede..abe48258f 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h @@ -566,6 +566,7 @@ typedef enum { XlogCopyBackendProcess, BarrierPreParseBackendProcess, DmsAuxiliaryProcess, + ExrtoRecyclerProcess, NUM_SINGLE_AUX_PROC, /* Sentry for auxiliary type with single thread. */ /* @@ -610,6 +611,7 @@ typedef enum { #define AmTsCompactionAuxiliaryProcess() (t_thrd.bootstrap_cxt.MyAuxProcType == TsCompactionAuxiliaryProcess) #define AmPageRedoWorker() (t_thrd.bootstrap_cxt.MyAuxProcType == PageRedoProcess) #define AmDmsReformProcProcess() (t_thrd.role == DMS_WORKER && t_thrd.dms_cxt.is_reform_proc) +#define AmErosRecyclerProcess() (t_thrd.bootstrap_cxt.MyAuxProcType == ExrtoRecyclerProcess) diff --git a/src/include/postgres.h b/src/include/postgres.h index 724c7895d..429da1baa 100644 --- a/src/include/postgres.h +++ b/src/include/postgres.h @@ -1026,4 +1026,6 @@ extern void exec_describe_statement_message(const char* stmt_name); extern void exec_get_ddl_params(StringInfo input_message); #endif +#define STRUCT_CONTAINER(type, membername, ptr) ((type *)((char *)(ptr)-offsetof(type, membername))) + #endif /* POSTGRES_H */ diff --git a/src/include/postmaster/postmaster.h b/src/include/postmaster/postmaster.h index 82e7ad1d2..0902eae8d 100755 --- a/src/include/postmaster/postmaster.h +++ b/src/include/postmaster/postmaster.h @@ -245,6 +245,9 @@ extern bool SetDBStateFileState(DbState state, bool optional); extern void GPCResetAll(); extern void initRandomState(TimestampTz start_time, TimestampTz stop_time); extern bool PMstateIsRun(void); +extern bool pm_state_is_startup(); +extern bool pm_state_is_recovery(); +extern bool pm_state_is_hot_standby(); extern ServerMode GetHaShmemMode(void); extern void InitProcessAndShareMemory(); extern void InitShmemForDcfCallBack(); diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h index 26d889f5d..ea1fb952f 100755 --- a/src/include/replication/walreceiver.h +++ b/src/include/replication/walreceiver.h @@ -41,6 +41,9 @@ #define IS_PAUSE_BY_TARGET_BARRIER 0x00000001 #define IS_CANCEL_LOG_CTRL 0x00000010 +#define IS_DISASTER_RECOVER_MODE \ + (static_cast(g_instance.attr.attr_common.stream_cluster_run_mode) == RUN_MODE_STANDBY) + #ifdef ENABLE_MULTIPLE_NODES #define AM_HADR_CN_WAL_RECEIVER (t_thrd.postmaster_cxt.HaShmData->is_cross_region && \ t_thrd.postmaster_cxt.HaShmData->current_mode == STANDBY_MODE && IS_PGXC_COORDINATOR) diff --git a/src/include/storage/buf/buf_internals.h b/src/include/storage/buf/buf_internals.h index b5a2d24b6..c233d89a1 100644 --- a/src/include/storage/buf/buf_internals.h +++ b/src/include/storage/buf/buf_internals.h @@ -57,6 +57,7 @@ */ #define BM_IN_MIGRATE (1U << 16) /* buffer is migrating */ #define BM_IS_META (1U << 17) +#define BM_IS_TMP_BUF (1U << 21) /* temp buf, can not write to disk */ #define BM_LOCKED (1U << 22) /* buffer header is locked */ #define BM_DIRTY (1U << 23) /* data needs writing */ #define BM_VALID (1U << 24) /* data is valid */ @@ -285,6 +286,23 @@ extern "C" { pg_atomic_write_u32(&(desc)->state, (s) & (~BM_LOCKED)); \ } while (0) +#define FIX_SEG_BUFFER_TAG(node, tag, rel_node, block_num) \ + do { \ + if (IsSegmentFileNode(node)) { \ + tag.rnode.relnode = rel_node; \ + tag.blocknum = block_num; \ + tag.rnode.bucketnode = SegmentBktId; \ + } \ + } while (0) + +#define FIX_BUFFER_DESC(buf, pblk) \ + do { \ + Assert(PhyBlockIsValid(*pblk)); \ + buf->seg_fileno = pblk->rel_node; \ + buf->seg_blockno = pblk->block; \ + buf->seg_lsn = pblk->lsn; \ + } while (0) + extern bool retryLockBufHdr(BufferDesc* desc, uint32* buf_state); /* * The PendingWriteback & WritebackContext structure are used to keep diff --git a/src/include/storage/buf/bufmgr.h b/src/include/storage/buf/bufmgr.h index 5581d23f6..61aa35e54 100644 --- a/src/include/storage/buf/bufmgr.h +++ b/src/include/storage/buf/bufmgr.h @@ -320,6 +320,7 @@ extern void DropRelFileNodeAllBuffersUsingScan(RelFileNode* rnode, int rnode_len extern void DropRelFileNodeOneForkAllBuffersUsingHash(HTAB *relfilenode_hashtbl); extern void DropDatabaseBuffers(Oid dbid); +extern void buffer_drop_exrto_standby_read_buffers(); extern BlockNumber PartitionGetNumberOfBlocksInFork(Relation relation, Partition partition, ForkNumber forkNum, bool estimate = false); @@ -423,4 +424,13 @@ extern void ReadBuffer_common_for_check(ReadBufferMode readmode, BufferDesc* buf const XLogPhyBlock *pblk, Block bufBlock); extern BufferDesc *RedoForOndemandExtremeRTOQuery(BufferDesc *bufHdr, char relpersistence, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode); +extern Buffer standby_read_buf(Relation reln, ForkNumber fork_num, BlockNumber block_num, ReadBufferMode mode, + BufferAccessStrategy strategy); +typedef struct SMgrRelationData *SMgrRelation; +BufferDesc *BufferAlloc(const RelFileNode &rel_file_node, char relpersistence, ForkNumber forkNum, BlockNumber blockNum, + BufferAccessStrategy strategy, bool *foundPtr, const XLogPhyBlock *pblk); +Buffer ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, BlockNumber blockNum, + ReadBufferMode mode, BufferAccessStrategy strategy, bool *hit, const XLogPhyBlock *pblk); +void buffer_in_progress_pop(); +void buffer_in_progress_push(); #endif diff --git a/src/include/storage/buf/bufpage.h b/src/include/storage/buf/bufpage.h index cb0fbcc98..5384af391 100644 --- a/src/include/storage/buf/bufpage.h +++ b/src/include/storage/buf/bufpage.h @@ -206,6 +206,7 @@ typedef HeapPageHeaderData* HeapPageHeader; #define PD_ENCRYPT_PAGE 0x0020 /* is a encryt cluster */ #define PD_CHECKSUM_FNV1A 0x0040 /* page checksum using FNV-1a hash */ #define PD_JUST_AFTER_FPW 0x0080 /* page just after redo full page write */ +#define PD_EXRTO_PAGE 0x0400 /* is a rto file page */ #define PD_TDE_PAGE 0x0100 /* there is TdePageInfo at the end of a page */ #define PD_VALID_FLAG_BITS 0x01FF /* OR of all valid pd_flags bits */ diff --git a/src/include/storage/nvm/nvm.h b/src/include/storage/nvm/nvm.h index 5501081f0..bc5db0fe2 100644 --- a/src/include/storage/nvm/nvm.h +++ b/src/include/storage/nvm/nvm.h @@ -28,7 +28,7 @@ void nvm_init(void); -BufferDesc *NvmBufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber fork_num, +BufferDesc *NvmBufferAlloc(const RelFileNode& rel_file_node, char relpersistence, ForkNumber fork_num, BlockNumber block_num, BufferAccessStrategy strategy, bool *found, const XLogPhyBlock *pblk); #endif diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h index bc122ff3f..b1abe2d8b 100755 --- a/src/include/storage/proc.h +++ b/src/include/storage/proc.h @@ -272,6 +272,8 @@ struct PGPROC { uint64 snap_refcnt_bitmap; #endif + XLogRecPtr exrto_read_lsn; /* calculate recycle lsn for read on standby in extreme rto */ + TimestampTz exrto_gen_snap_time; LWLock* subxidsLock; struct XidCache subxids; /* cache for subtransaction XIDs */ diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h index 91f114628..290d88c50 100755 --- a/src/include/storage/procarray.h +++ b/src/include/storage/procarray.h @@ -115,6 +115,8 @@ extern VirtualTransactionId *GetConflictingVirtualXIDs(TransactionId limitXmin, CommitSeqNo limitXminCSN = InvalidCommitSeqNo, TransactionId* xminArray = NULL); extern ThreadId CancelVirtualTransaction(const VirtualTransactionId& vxid, ProcSignalReason sigmode); +extern bool proc_array_cancel_conflicting_proc(TransactionId latest_removed_xid, + bool reach_max_check_times); extern bool MinimumActiveBackends(int min); extern int CountDBBackends(Oid database_oid); diff --git a/src/include/storage/smgr/relfilenode.h b/src/include/storage/smgr/relfilenode.h index 89a3725c7..c06955bd0 100644 --- a/src/include/storage/smgr/relfilenode.h +++ b/src/include/storage/smgr/relfilenode.h @@ -116,7 +116,7 @@ typedef struct RelFileNodeV2 { } RelFileNodeV2; -#define IsSegmentFileNode(rnode) ((rnode).bucketNode > InvalidBktId) +#define IsSegmentFileNode(rnode) ((rnode).bucketNode > InvalidBktId && (rnode).spcNode != EXRTO_BLOCK_INFO_SPACE_OID) #define IsHeapFileNode(rnode) (!IsSegmentFileNode(rnode)) #define IsSegmentPhysicalRelNode(rNode) (IsSegmentFileNode(rNode) && (rNode).relNode <= 5) diff --git a/src/include/storage/smgr/smgr.h b/src/include/storage/smgr/smgr.h index 9a68cb4be..9acbb1643 100644 --- a/src/include/storage/smgr/smgr.h +++ b/src/include/storage/smgr/smgr.h @@ -125,12 +125,20 @@ enum SMGR_READ_STATUS { #define UNDO_DB_OID (9) #define UNDO_SLOT_DB_OID (10) +#define EXRTO_BASE_PAGE_SPACE_OID (6) +#define EXRTO_LSN_INFO_SPACE_OID (7) +#define EXRTO_BLOCK_INFO_SPACE_OID (8) +#define EXRTO_FORK_NUM 3 + #define MD_MANAGER (0) #define UNDO_MANAGER (1) #define SEGMENT_MANAGER (2) +#define EXRTO_MANAGER (3) #define IS_UNDO_RELFILENODE(rnode) ((rnode).dbNode == UNDO_DB_OID || (rnode).dbNode == UNDO_SLOT_DB_OID) - +#define IS_EXRTO_RELFILENODE(rnode) ((rnode).spcNode == EXRTO_BASE_PAGE_SPACE_OID || \ + (rnode).spcNode == EXRTO_LSN_INFO_SPACE_OID || \ + (rnode).spcNode == EXRTO_BLOCK_INFO_SPACE_OID) /* * On Windows, we have to interpret EACCES as possibly meaning the same as * ENOENT, because if a file is unlinked-but-not-yet-gone on that platform, @@ -250,4 +258,16 @@ extern void partition_create_new_storage(Relation rel, Partition part, const Rel extern ScalarToDatum GetTransferFuncByTypeOid(Oid attTypeOid); extern bool check_unlink_rel_hashtbl(RelFileNode rnode, ForkNumber forknum); +/* storage_exrto_file.cpp */ +void exrto_init(void); +void exrto_close(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum); +bool exrto_exists(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum); +void exrto_unlink(const RelFileNodeBackend& rnode, ForkNumber forknum, bool is_redo, BlockNumber blocknum); +void exrto_extend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, char *buffer, bool skip_fsync); +SMGR_READ_STATUS exrto_read(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, char *buffer); +void exrto_write(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, const char *buffer, bool skip_fsync); +BlockNumber exrto_nblocks(SMgrRelation reln, ForkNumber forknum); +void exrto_truncate(SMgrRelation reln, ForkNumber forknum, BlockNumber nblocks); +void exrto_writeback(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, BlockNumber nblocks); + #endif /* SMGR_H */ diff --git a/src/include/utils/snapshot.h b/src/include/utils/snapshot.h index de5b0d85a..1c38951e6 100644 --- a/src/include/utils/snapshot.h +++ b/src/include/utils/snapshot.h @@ -265,6 +265,32 @@ typedef struct SnapshotData { GTM_SnapshotType gtm_snapshot_type; } SnapshotData; +typedef struct ExrtoSnapshotData* ExrtoSnapshot; + +typedef struct ExrtoSnapshotData { + /* + * The remaining fields are used only for MVCC snapshots, and are normally + * just zeroes in special snapshots. (But xmin and xmax are used + * specially by HeapTupleSatisfiesDirty.) + * + * An MVCC snapshot can never see the effects of XIDs >= xmax. It can see + * the effects of all older XIDs except those listed in the snapshot. xmin + * is stored as an optimization to avoid needing to search the XID arrays + * for most tuples. + */ + TransactionId xmin; /* all XID < xmin are visible to me */ + TransactionId xmax; /* all XID >= xmax are invisible to me */ + + /* + * This snapshot can see the effects of all transactions with CSN <= + * snapshotcsn. + */ + CommitSeqNo snapshot_csn; + + XLogRecPtr read_lsn; /* xact lsn when generate snapshot */ + TimestampTz gen_snap_time; +} ExrtoSnapshotData; + /* * Result codes for AM API tuple_{update,delete,lock}, and for visibility. */ diff --git a/src/test/regress/output/recovery_2pc_tools.source b/src/test/regress/output/recovery_2pc_tools.source index 6b0ac7591..2fd58569c 100644 --- a/src/test/regress/output/recovery_2pc_tools.source +++ b/src/test/regress/output/recovery_2pc_tools.source @@ -78,6 +78,7 @@ select name,vartype,unit,min_val,max_val from pg_settings where name <> 'qunit_c backtrace_min_messages | enum | | | backwrite_quantity | integer | 8kB | 128 | 131072 basebackup_timeout | integer | s | 0 | 2147483647 + base_page_saved_interval | integer | | 5 | 2000 bbox_blanklist_items | string | | | bbox_dump_count | integer | | 1 | 20 bbox_dump_path | string | | | @@ -487,6 +488,8 @@ select name,vartype,unit,min_val,max_val from pg_settings where name <> 'qunit_c max_size_for_xlog_prune | integer | kB | 0 | 2147483647 max_stack_depth | integer | kB | 100 | 2147483647 max_standby_archive_delay | integer | ms | -1 | 2147483647 + max_standby_base_page_size | int64 | | 0 | 576460752303423487 + max_standby_lsn_info_size | int64 | | 0 | 576460752303423487 max_standby_streaming_delay | integer | ms | -1 | 2147483647 max_sync_workers_per_subscription | integer | | 0 | 262143 max_undo_workers | integer | | 1 | 100 @@ -660,6 +663,9 @@ select name,vartype,unit,min_val,max_val from pg_settings where name <> 'qunit_c ss_txnstatus_cache_size | integer | | 0 | 524288 ss_work_thread_count | integer | | 16 | 128 standard_conforming_strings | bool | | | + standby_force_recyle_ratio | real | | 0 | 1 + standby_max_query_time | integer | s | 0 | 86400 + standby_recycle_interval | integer | s | 0 | 86400 standby_shared_buffers_fraction | real | | 0.1 | 1 statement_timeout | integer | ms | 0 | 2147483647 stats_temp_directory | string | | | From 25eaab5467abd6e2d86407f9abf632163366f3d6 Mon Sep 17 00:00:00 2001 From: "changying.yue" Date: Wed, 30 Aug 2023 06:23:08 +0000 Subject: [PATCH 182/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dbool=E7=B1=BB?= =?UTF-8?q?=E5=9E=8B=E5=8F=98=E9=87=8Fand=E6=93=8D=E4=BD=9C=E7=AC=A6?= =?UTF-8?q?=E5=8F=B7=E4=BD=BF=E7=94=A8=E9=94=99=E8=AF=AF=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/pl/plpgsql/src/pl_exec.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/pl/plpgsql/src/pl_exec.cpp b/src/common/pl/plpgsql/src/pl_exec.cpp index f1de71665..fde5e6e68 100644 --- a/src/common/pl/plpgsql/src/pl_exec.cpp +++ b/src/common/pl/plpgsql/src/pl_exec.cpp @@ -6859,13 +6859,13 @@ static int exec_stmt_execsql(PLpgSQL_execstate* estate, PLpgSQL_stmt_execsql* st * to enforce strictness. */ if (stmt->into) { - if (!stmt->mod_stmt & !stmt->bulk_collect) { + if (!stmt->mod_stmt && !stmt->bulk_collect) { if (!DB_IS_CMPT(PG_FORMAT | B_FORMAT) || SELECT_INTO_RETURN_NULL == 0) { stmt->strict = true; } } #ifdef ENABLE_MULTIPLE_NODES - if (!stmt->mod_stmt & !stmt->bulk_collect) { + if (!stmt->mod_stmt && !stmt->bulk_collect) { stmt->strict = true; } #endif From a6980372419b9e9815df35cbdfe07c3f069e685d Mon Sep 17 00:00:00 2001 From: jiwenke Date: Wed, 30 Aug 2023 16:00:46 +0800 Subject: [PATCH 183/304] =?UTF-8?q?=E6=9C=AA=E5=BC=80=E5=90=AFDSS=E6=97=B6?= =?UTF-8?q?=EF=BC=8Cndpplugin=E6=8F=92=E4=BB=B6=E5=8A=A0=E8=BD=BD=E6=A0=A1?= =?UTF-8?q?=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- contrib/ndpplugin/ndpplugin.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/contrib/ndpplugin/ndpplugin.cpp b/contrib/ndpplugin/ndpplugin.cpp index b80337b87..3dd041a82 100644 --- a/contrib/ndpplugin/ndpplugin.cpp +++ b/contrib/ndpplugin/ndpplugin.cpp @@ -1464,6 +1464,11 @@ void _PG_init(void) { ereport(DEBUG2, (errmsg("init ndpplugin."))); + if (!ENABLE_DSS) { + ereport(DEBUG2, (errmsg("ndpplugin is not support while DMS and DSS disable."))); + return; + } + pthread_mutex_lock(&g_ndp_instance.mutex); #ifdef GlobalCache From 986c4317ca60414a1cb29620e6eae501605d7208 Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Wed, 30 Aug 2023 16:50:12 +0800 Subject: [PATCH 184/304] =?UTF-8?q?830dss=E6=8E=A8=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/ddes/ddes_commit_id | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gausskernel/ddes/ddes_commit_id b/src/gausskernel/ddes/ddes_commit_id index e2f8422e9..d327a12eb 100644 --- a/src/gausskernel/ddes/ddes_commit_id +++ b/src/gausskernel/ddes/ddes_commit_id @@ -1,3 +1,3 @@ dms_commit_id=d3b22f7482078513b98dcb3241e17b26962e8a36 -dss_commit_id=45ae7916cbdda2b2e64c02811db30df565cf34fa +dss_commit_id=f8adb3e023c05fbed5c90a4449a2e8263f5297fd cbb_commit_id=41cadda4643656bcd856bd37d2b6908e44a5d46c \ No newline at end of file From b17e58e7562784e9455a4f0f0cfa845d08b9aff5 Mon Sep 17 00:00:00 2001 From: "arcoalien@qq.com" Date: Wed, 30 Aug 2023 19:43:11 +0800 Subject: [PATCH 185/304] =?UTF-8?q?=E4=BC=98=E5=8C=96cm=5Fagent=E8=8E=B7?= =?UTF-8?q?=E5=8F=96=E5=BF=AB=E7=85=A7=E9=80=BB=E8=BE=91=EF=BC=8C=E4=B8=8D?= =?UTF-8?q?=E8=B5=B0dms=EF=BC=8C=E5=87=8F=E5=B0=91cm=5Fagent=E8=8E=B7?= =?UTF-8?q?=E5=8F=96=E7=8A=B6=E6=80=81=E5=BC=82=E5=B8=B8=E7=9A=84=E5=9C=BA?= =?UTF-8?q?=E6=99=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/ddes/adapter/ss_transaction.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/gausskernel/ddes/adapter/ss_transaction.cpp b/src/gausskernel/ddes/adapter/ss_transaction.cpp index d03c2d481..a25033bb7 100644 --- a/src/gausskernel/ddes/adapter/ss_transaction.cpp +++ b/src/gausskernel/ddes/adapter/ss_transaction.cpp @@ -99,6 +99,12 @@ Snapshot SSGetSnapshotData(Snapshot snapshot) return NULL; } + /* For cm agent, it only query the system status using the parameter in memory. So don't need MVCC */ + if (u_sess->libpq_cxt.IsConnFromCmAgent) { + snapshot = SnapshotNow; + return snapshot; + } + if (!ENABLE_SS_BCAST_SNAPSHOT || (g_instance.dms_cxt.latest_snapshot_xmax == InvalidTransactionId && t_thrd.dms_cxt.latest_snapshot_xmax == InvalidTransactionId)) { From d93d27881937c10cd1ca121d952486c04b136eba Mon Sep 17 00:00:00 2001 From: jiwenke Date: Thu, 31 Aug 2023 15:18:47 +0800 Subject: [PATCH 186/304] =?UTF-8?q?dss=208.31=E6=8E=A8=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/ddes/ddes_commit_id | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gausskernel/ddes/ddes_commit_id b/src/gausskernel/ddes/ddes_commit_id index d327a12eb..8013d87a3 100644 --- a/src/gausskernel/ddes/ddes_commit_id +++ b/src/gausskernel/ddes/ddes_commit_id @@ -1,3 +1,3 @@ dms_commit_id=d3b22f7482078513b98dcb3241e17b26962e8a36 -dss_commit_id=f8adb3e023c05fbed5c90a4449a2e8263f5297fd +dss_commit_id=1244c7ee0ec493b9cdf13b1ea86335ebf87b3eda cbb_commit_id=41cadda4643656bcd856bd37d2b6908e44a5d46c \ No newline at end of file From 739949b4adb1e6432c14538e96f98679a7019aa8 Mon Sep 17 00:00:00 2001 From: vastdata-xyzr Date: Thu, 31 Aug 2023 17:46:24 +0800 Subject: [PATCH 187/304] =?UTF-8?q?=E5=88=A0=E9=99=A4procarray.cpp?= =?UTF-8?q?=E4=B8=AD=E6=B2=A1=E6=9C=89=E4=BD=BF=E7=94=A8=E7=9A=84=E5=8F=98?= =?UTF-8?q?=E9=87=8F=E5=AE=9A=E4=B9=89=EF=BC=8C=E6=B6=88=E9=99=A4=E7=BC=96?= =?UTF-8?q?=E8=AF=91=E8=AD=A6=E5=91=8A=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/storage/ipc/procarray.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/gausskernel/storage/ipc/procarray.cpp b/src/gausskernel/storage/ipc/procarray.cpp index b23a6b590..39e78271b 100755 --- a/src/gausskernel/storage/ipc/procarray.cpp +++ b/src/gausskernel/storage/ipc/procarray.cpp @@ -2372,7 +2372,6 @@ Snapshot GetSnapshotData(Snapshot snapshot, bool force_local_snapshot) /* Check whether there's a standby requiring an older xmin when dms is enabled. */ if (SS_NORMAL_PRIMARY && SS_REPLICATION_MAIN_STANBY_NODE) { - ss_xmin_info_t* xmin_info = &g_instance.dms_cxt.SSXminInfo; uint64 global_xmin = SSGetGlobalOldestXmin(u_sess->utils_cxt.RecentGlobalXmin); u_sess->utils_cxt.RecentGlobalXmin = global_xmin; } From 01fd8e022a7841dbe503342e84be1165f8d6a82c Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Thu, 31 Aug 2023 18:43:06 +0800 Subject: [PATCH 188/304] =?UTF-8?q?831dss=E6=8E=A8=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/ddes/ddes_commit_id | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gausskernel/ddes/ddes_commit_id b/src/gausskernel/ddes/ddes_commit_id index 8013d87a3..e88d16fc9 100644 --- a/src/gausskernel/ddes/ddes_commit_id +++ b/src/gausskernel/ddes/ddes_commit_id @@ -1,3 +1,3 @@ dms_commit_id=d3b22f7482078513b98dcb3241e17b26962e8a36 -dss_commit_id=1244c7ee0ec493b9cdf13b1ea86335ebf87b3eda +dss_commit_id=e8fbe469e1b320c34684ba7c15c5fbc5d5208cf8 cbb_commit_id=41cadda4643656bcd856bd37d2b6908e44a5d46c \ No newline at end of file From c78ba2f5d7bd4a44e67550698a9b2f67893a532b Mon Sep 17 00:00:00 2001 From: chenxiaobin19 <1025221611@qq.com> Date: Fri, 1 Sep 2023 11:48:50 +0800 Subject: [PATCH 189/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dskiplsn=E5=81=B6?= =?UTF-8?q?=E7=8E=B0=E5=A4=B1=E6=95=88&&=E5=88=86=E5=8C=BA=E8=A1=A8?= =?UTF-8?q?=E5=A4=84=E7=90=86=E5=86=B2=E7=AA=81=E5=9C=BA=E6=99=AF=E5=AF=B9?= =?UTF-8?q?=E8=B1=A1=E6=9C=AA=E9=87=8A=E6=94=BE=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../storage/replication/logical/worker.cpp | 48 ++++++---- src/test/subscription/testcase/bugs.sh | 90 +++++++++++++++++++ 2 files changed, 122 insertions(+), 16 deletions(-) diff --git a/src/gausskernel/storage/replication/logical/worker.cpp b/src/gausskernel/storage/replication/logical/worker.cpp index 54a903fe0..0b6cc2aeb 100644 --- a/src/gausskernel/storage/replication/logical/worker.cpp +++ b/src/gausskernel/storage/replication/logical/worker.cpp @@ -116,7 +116,7 @@ static void apply_dispatch(StringInfo s); static void apply_handle_conninfo(StringInfo s); static void UpdateConninfo(char* standbysInfo); static Oid find_conflict_tuple(EState *estate, TupleTableSlot *remoteslot, TupleTableSlot *localslot, - FakeRelationPartition *fakeRelInfo, TupleTableSlot *originslot = NULL); + TupleTableSlot *originslot = NULL); static void IsSkippingChanges(XLogRecPtr finish_lsn); static void StopSkippingChanges(); @@ -599,17 +599,26 @@ static Oid GetRelationIdentityOrPK(Relation rel) * to be updated, ignore it. */ Oid find_conflict_tuple(EState *estate, TupleTableSlot *remoteslot, TupleTableSlot *localslot, - FakeRelationPartition *fakeRelInfo, TupleTableSlot *originslot) + TupleTableSlot *originslot) { Oid replidxoid = InvalidOid; bool found = false; ResultRelInfo* relinfo = estate->es_result_relation_info; + FakeRelationPartition fakeRelInfo; /* Check the replica identity index first */ replidxoid = RelationGetReplicaIndex(relinfo->ri_RelationDesc); if (OidIsValid(replidxoid)) { found = RelationFindReplTuple(estate, relinfo->ri_RelationDesc, replidxoid, LockTupleExclusive, remoteslot, - localslot, fakeRelInfo); + localslot, &fakeRelInfo); + + /* Cleanup. */ + if (fakeRelInfo.needRleaseDummyRel && fakeRelInfo.partRel) { + releaseDummyRelation(&fakeRelInfo.partRel); + } + if (fakeRelInfo.partList) { + releasePartitionList(relinfo->ri_RelationDesc, &fakeRelInfo.partList, NoLock); + } if (found) { if (originslot != NULL && ItemPointerCompare(tableam_tops_get_t_self(relinfo->ri_RelationDesc, @@ -640,7 +649,15 @@ Oid find_conflict_tuple(EState *estate, TupleTableSlot *remoteslot, TupleTableSl } found = RelationFindReplTuple(estate, relinfo->ri_RelationDesc, idxoid, LockTupleExclusive, remoteslot, - localslot, fakeRelInfo); + localslot, &fakeRelInfo); + + /* Cleanup. */ + if (fakeRelInfo.needRleaseDummyRel && fakeRelInfo.partRel) { + releaseDummyRelation(&fakeRelInfo.partRel); + } + if (fakeRelInfo.partList) { + releasePartitionList(relinfo->ri_RelationDesc, &fakeRelInfo.partList, NoLock); + } if (found) { if (originslot != NULL && ItemPointerCompare(tableam_tops_get_t_self(relinfo->ri_RelationDesc, @@ -673,12 +690,12 @@ static void apply_handle_insert(StringInfo s) EPQState epqstate; Oid conflictIndexOid = InvalidOid; + ensure_transaction(); + if (t_thrd.applyworker_cxt.isSkipTransaction) { return; } - ensure_transaction(); - relid = logicalrep_read_insert(s, &newtup); rel = logicalrep_rel_open(relid, RowExclusiveLock); if (!should_apply_changes_for_rel(rel)) { @@ -707,8 +724,11 @@ static void apply_handle_insert(StringInfo s) ExecOpenIndices(estate->es_result_relation_info, false); + /* Get fake relation and partition for patitioned table */ + GetFakeRelAndPart(estate, rel->localrel, remoteslot, &fakeRelInfo); + if (t_thrd.applyworker_cxt.curWorker->needCheckConflict && - (conflictIndexOid = find_conflict_tuple(estate, remoteslot, localslot, &fakeRelInfo)) != InvalidOid) { + (conflictIndexOid = find_conflict_tuple(estate, remoteslot, localslot)) != InvalidOid) { StringInfoData localtup, remotetup; initStringInfo(&localtup); tuple_to_stringinfo(rel->localrel, &localtup, RelationGetDescr(rel->localrel), @@ -760,9 +780,6 @@ static void apply_handle_insert(StringInfo s) } else { PG_TRY(); { - /* Get fake relation and partition for patitioned table */ - GetFakeRelAndPart(estate, rel->localrel, remoteslot, &fakeRelInfo); - /* Do the insert. */ ExecSimpleRelationInsert(estate, remoteslot, &fakeRelInfo); } @@ -876,12 +893,12 @@ static void apply_handle_update(StringInfo s) FakeRelationPartition fakeRelInfo; Oid conflictIndexOid = InvalidOid; + ensure_transaction(); + if (t_thrd.applyworker_cxt.isSkipTransaction) { return; } - ensure_transaction(); - relid = logicalrep_read_update(s, &has_oldtup, &oldtup, &newtup); rel = logicalrep_rel_open(relid, RowExclusiveLock); if (!should_apply_changes_for_rel(rel)) { @@ -960,8 +977,7 @@ static void apply_handle_update(StringInfo s) MemoryContextSwitchTo(oldctx); if (t_thrd.applyworker_cxt.curWorker->needCheckConflict && - (conflictIndexOid = find_conflict_tuple(estate, remoteslot, conflictLocalSlot, &fakeRelInfo, localslot)) - != InvalidOid) { + (conflictIndexOid = find_conflict_tuple(estate, remoteslot, conflictLocalSlot, localslot)) != InvalidOid) { StringInfoData localtup, remotetup; initStringInfo(&localtup); tuple_to_stringinfo(rel->localrel, &localtup, RelationGetDescr(rel->localrel), @@ -1075,12 +1091,12 @@ static void apply_handle_delete(StringInfo s) MemoryContext oldctx; FakeRelationPartition fakeRelInfo; + ensure_transaction(); + if (t_thrd.applyworker_cxt.isSkipTransaction) { return; } - ensure_transaction(); - relid = logicalrep_read_delete(s, &oldtup); rel = logicalrep_rel_open(relid, RowExclusiveLock); if (!should_apply_changes_for_rel(rel)) { diff --git a/src/test/subscription/testcase/bugs.sh b/src/test/subscription/testcase/bugs.sh index ec35e5473..5eaa09491 100644 --- a/src/test/subscription/testcase/bugs.sh +++ b/src/test/subscription/testcase/bugs.sh @@ -8,6 +8,8 @@ function test_1() { echo "create database and tables." exec_sql $db $pub_node1_port "CREATE DATABASE $case_db" exec_sql $db $sub_node1_port "CREATE DATABASE $case_db" + + # BUG1: coredump when apply null value into not null column. # Create some preexisting content on publisher exec_sql $case_db $pub_node1_port "CREATE TABLE tab_rep (a int primary key, b text)" exec_sql $case_db $pub_node1_port "INSERT INTO tab_rep VALUES (1)" @@ -43,6 +45,94 @@ function test_1() { done echo "check failing row log success" + + exec_sql $case_db $sub_node1_port "DROP SUBSCRIPTION IF EXISTS tap_sub;TRUNCATE TABLE tab_rep" + exec_sql $case_db $pub_node1_port "DROP PUBLICATION IF EXISTS tap_pub;TRUNCATE TABLE tab_rep" + + # BUG2: skiplsn does not work occasionally + # Setup logical replication + echo "create publication and subscription." + publisher_connstr="port=$pub_node1_port host=$g_local_ip dbname=$case_db user=$username password=$passwd" + exec_sql $case_db $pub_node1_port "CREATE PUBLICATION tap_pub FOR ALL TABLES" + exec_sql $case_db $sub_node1_port "CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr' PUBLICATION tap_pub with (copy_data=false); ALTER SUBSCRIPTION tap_sub DISABLE" + + exec_sql $case_db $pub_node1_port "INSERT INTO tab_rep VALUES (1, 'pub1'); INSERT INTO tab_rep VALUES (2, 'sub2');" + dumpfile=$(exec_sql $case_db $pub_node1_port "select gs_xlogdump_xid(xmin) from tab_rep where a = 1;") + skiplsn=$(grep 'start_lsn' $dumpfile | sed -n '5p' | awk '{print $2}') + + exec_sql $case_db $sub_node1_port "alter subscription tap_sub set (skiplsn = '$skiplsn')" + exec_sql $case_db $sub_node1_port "alter subscription tap_sub enable" + + wait_for_catchup $case_db $pub_node1_port "tap_sub" + + if [ "$(exec_sql $case_db $sub_node1_port "SELECT * FROM tab_rep")" = "2|sub2" ]; then + echo "check data skip success" + else + echo "$failed_keyword when check data skip" + exit 1 + fi + + exec_sql $case_db $sub_node1_port "DROP SUBSCRIPTION IF EXISTS tap_sub;DROP TABLE tab_rep" + exec_sql $case_db $pub_node1_port "DROP PUBLICATION IF EXISTS tap_pub;DROP TABLE tab_rep" + + # BUG3: partition relation not closed when handle conflict + # Create partition table + ddl=" +create table t_pubsub_0349( + id int primary key constraint id_nn not null, + use_filename varchar(20), + filename varchar2(255) +)partition by range(id)( + partition p1 values less than(30), + partition p2 values less than(60), + partition p3 values less than(90), + partition p4 values less than(maxvalue));" + exec_sql $case_db $pub_node1_port "$ddl" + exec_sql $case_db $sub_node1_port "$ddl" + + echo "create publication and subscription." + publisher_connstr="port=$pub_node1_port host=$g_local_ip dbname=$case_db user=$username password=$passwd" + exec_sql $case_db $pub_node1_port "CREATE PUBLICATION tap_pub FOR ALL TABLES" + exec_sql $case_db $sub_node1_port "CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr' PUBLICATION tap_pub" + + wait_for_subscription_sync $case_db $sub_node1_port + + exec_sql $case_db $sub_node1_port "ALTER SYSTEM SET subscription_conflict_resolution = apply_remote" + + exec_sql $case_db $sub_node1_port "INSERT INTO t_pubsub_0349 VALUES (1, 'a', 'a');" + exec_sql $case_db $pub_node1_port "INSERT INTO t_pubsub_0349 VALUES (1, 'a', 'c');" + + wait_for_catchup $case_db $pub_node1_port "tap_sub" + + if [ "$(exec_sql $case_db $sub_node1_port "SELECT * FROM t_pubsub_0349")" = "1|a|c" ]; then + echo "check insert conflict handle success" + else + echo "$failed_keyword when check insert conflict handle" + exit 1 + fi + + exec_sql $case_db $sub_node1_port "INSERT INTO t_pubsub_0349 VALUES (2, 'a', 'a');" + exec_sql $case_db $pub_node1_port "UPDATE t_pubsub_0349 SET id = 2 WHERE id = 1;" + + wait_for_catchup $case_db $pub_node1_port "tap_sub" + + if [ "$(exec_sql $case_db $sub_node1_port "SELECT * FROM t_pubsub_0349")" = "2|a|c" ]; then + echo "check update conflict handle success" + else + echo "$failed_keyword when check update conflict handle" + exit 1 + fi + + logfile=$(get_log_file "sub_datanode1") + leakstr=$(grep 'partcache reference leak' $logfile -m 1) + if [ -z "$leakstr" ]; then + echo "check relation close success" + else + echo "$failed_keyword when check relation close" + exit 1 + fi + + exec_sql $case_db $sub_node1_port "ALTER SYSTEM SET subscription_conflict_resolution = error" } function tear_down() { From 5978c74cb40039e5b3a0da2e20b677a19cd3915a Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Fri, 1 Sep 2023 15:25:32 +0800 Subject: [PATCH 190/304] =?UTF-8?q?build=20check=E9=80=82=E9=85=8D?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E5=90=88=E4=B8=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/pg_ctl/pg_ctl.cpp | 3 +- src/bin/pg_rewind/parsexlog.cpp | 21 +- src/bin/pg_rewind/pg_rewind.cpp | 4 +- .../process/postmaster/postmaster.cpp | 7 +- .../storage/access/transam/xlogreader.cpp | 194 ++++++++++++++++++ src/include/access/xlogreader.h | 2 + 6 files changed, 205 insertions(+), 26 deletions(-) diff --git a/src/bin/pg_ctl/pg_ctl.cpp b/src/bin/pg_ctl/pg_ctl.cpp index 67fcd8a55..57bf63823 100755 --- a/src/bin/pg_ctl/pg_ctl.cpp +++ b/src/bin/pg_ctl/pg_ctl.cpp @@ -4920,7 +4920,8 @@ static bool DoBuildCheck(uint32 term) buildSuccess = build_check_main(term); if (!buildSuccess) { - pg_log(PG_WARNING, _("%s failed(%s), need to do full build\n"), BuildModeToString(build_mode), pg_data); + pg_log(PG_WARNING, _("Build check result : full build\n"), BuildModeToString(build_mode), pg_data); + pg_log(PG_WARNING, _("%s failed(%s).\n"), BuildModeToString(build_mode), pg_data); } else { pg_log(PG_WARNING, _("%s completed(%s).\n"), BuildModeToString(build_mode), pg_data); } diff --git a/src/bin/pg_rewind/parsexlog.cpp b/src/bin/pg_rewind/parsexlog.cpp index 62ee0e870..24b183cf1 100644 --- a/src/bin/pg_rewind/parsexlog.cpp +++ b/src/bin/pg_rewind/parsexlog.cpp @@ -159,10 +159,7 @@ BuildErrorCode findCommonCheckpoint(const char* datadir, TimeLineID tli, XLogRec * local max lsn must be exists, or change to full build. */ if (ss_instance_config.dss.enable_dss) { - ret = snprintf_s(dssxlogdir, MAXPGPATH, MAXPGPATH - 1, "%s/%s%d", - ss_instance_config.dss.vgname, XLOGDIR, ss_instance_config.dss.instance_id); - securec_check_ss_c(ret, "", ""); - max_lsn = FindMaxLSN(datadir_target, returnmsg, XLOG_READER_MAX_MSGLENTH, &maxLsnCrc, NULL, NULL, dssxlogdir); + max_lsn = SSFindMaxLSN(datadir_target, returnmsg, XLOG_READER_MAX_MSGLENTH, &maxLsnCrc, ss_instance_config.dss.vgname); } else { max_lsn = FindMaxLSN(datadir_target, returnmsg, XLOG_READER_MAX_MSGLENTH, &maxLsnCrc); } @@ -200,21 +197,7 @@ BuildErrorCode findCommonCheckpoint(const char* datadir, TimeLineID tli, XLogRec uint8 info; if (ss_instance_config.dss.enable_dss) { - struct dirent *entry; - DIR* dssdir = opendir(ss_instance_config.dss.vgname); - while (dssdir != NULL && (entry = readdir(dssdir)) != NULL) { - if (strncmp(entry->d_name, "pg_xlog", strlen("pg_xlog")) == 0) { - ret = snprintf_s(dssxlogdir, MAXPGPATH, MAXPGPATH - 1, "%s/%s", ss_instance_config.dss.vgname, entry->d_name); - securec_check_ss_c(ret, "", ""); - record = XLogReadRecord(xlogreader, searchptr, &errormsg, true, dssxlogdir); - if (record != NULL) { - break; - } - } else { - continue; - } - } - closedir(dssdir); + record = XLogReadRecordFromAllDir(ss_instance_config.dss.vgname, xlogreader, searchptr, &errormsg); } else { record = XLogReadRecord(xlogreader, searchptr, &errormsg); } diff --git a/src/bin/pg_rewind/pg_rewind.cpp b/src/bin/pg_rewind/pg_rewind.cpp index bcac9409c..d4a30ff7c 100755 --- a/src/bin/pg_rewind/pg_rewind.cpp +++ b/src/bin/pg_rewind/pg_rewind.cpp @@ -1049,9 +1049,9 @@ BuildErrorCode do_build_check(const char* pgdata, const char* connstr, char* sys pg_log(PG_PROGRESS, "find diverge point success\n"); if (chkptrec == ControlFile_target.checkPoint) { - pg_log(PG_PROGRESS, "do not need to build\n"); + pg_log(PG_PROGRESS, "Build check result : needless build\n"); } else { - pg_log(PG_PROGRESS, "need to do incremental build\n"); + pg_log(PG_PROGRESS, "Build check result : incremental build\n"); } /* Disconnect from remote server */ if (connstr_source != NULL) { diff --git a/src/gausskernel/process/postmaster/postmaster.cpp b/src/gausskernel/process/postmaster/postmaster.cpp index ea5d789a7..9c3d5b029 100644 --- a/src/gausskernel/process/postmaster/postmaster.cpp +++ b/src/gausskernel/process/postmaster/postmaster.cpp @@ -12699,10 +12699,9 @@ const char* wal_get_db_state_string(DbState db_state) static ServerMode get_cur_mode(void) { - if (SS_REPLICATION_DORADO_CLUSTER) { - if (SS_REPLICATION_MAIN_STANBY_NODE) { - return STANDBY_MODE; - } + if (SS_REPLICATION_MAIN_STANBY_NODE) { + return STANDBY_MODE; + } else if (ENABLE_DMS) { return SS_OFFICIAL_PRIMARY ? PRIMARY_MODE : STANDBY_MODE; } return t_thrd.postmaster_cxt.HaShmData->current_mode; diff --git a/src/gausskernel/storage/access/transam/xlogreader.cpp b/src/gausskernel/storage/access/transam/xlogreader.cpp index 8b88abfe4..966cd3911 100644 --- a/src/gausskernel/storage/access/transam/xlogreader.cpp +++ b/src/gausskernel/storage/access/transam/xlogreader.cpp @@ -48,6 +48,14 @@ static THR_LOCAL int xlogreadfd = -1; static THR_LOCAL XLogSegNo xlogreadsegno = 0; #endif +#define CLOSE_FD(fd) \ + do { \ + if (fd > 0) { \ + close(fd); \ + fd = -1; \ + } \ + } while (0) + bool ValidXLogPageHeader(XLogReaderState *state, XLogRecPtr recptr, XLogPageHeader hdr); static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen, char* xlog_path); void ResetDecoder(XLogReaderState *state); @@ -1318,6 +1326,192 @@ int SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, in return XLOG_BLCKSZ; } +XLogRecord* XLogReadRecordFromAllDir(char* dirPath, XLogReaderState *xlogReader, XLogRecPtr curLsn, char** errorMsg) +{ + DIR* dir = opendir(dirPath); + struct dirent* entry = NULL; + char xlogDirStr[MAXPGPATH]; + errno_t rc = EOK; + XLogRecord* record = NULL; + + while (dir != NULL && (entry = readdir(dir)) != NULL) { + if (strncmp(entry->d_name, "pg_xlog", strlen("pg_xlog")) == 0) { + rc = snprintf_s(xlogDirStr, MAXPGPATH, MAXPGPATH - 1, "%s/%s", dirPath, entry->d_name); + securec_check_ss_c(rc, "", ""); + record = XLogReadRecord(xlogReader, curLsn, errorMsg, true, xlogDirStr); + if (record != NULL) { + break; + } else { + CLOSE_FD(xlogreadfd); + } + } + } + (void)closedir(dir); + return record; +} + +void FindMaxXlogFileName(char* dirPath, char* maxXLogFileName) +{ + DIR* dir = opendir(dirPath); + struct dirent* entry = NULL; + char xlogDirStr[MAXPGPATH]; + errno_t rc = EOK; + + while (dir != NULL && (entry = readdir(dir)) != NULL) { + if (strncmp(entry->d_name, "pg_xlog", strlen("pg_xlog")) == 0) { + rc = snprintf_s(xlogDirStr, MAXPGPATH, MAXPGPATH - 1, "%s/%s", dirPath, entry->d_name); + securec_check_ss_c(rc, "", ""); + DIR* subDir = opendir(xlogDirStr); + struct dirent* subDirEntry = NULL; + while (subDir != NULL && (subDirEntry = readdir(subDir)) != NULL) { + if (strlen(subDirEntry->d_name) == 24 && strspn(subDirEntry->d_name, "0123456789ABCDEF") == 24 && + (strlen(maxXLogFileName) == 0 || strcmp(maxXLogFileName, subDirEntry->d_name) < 0)) { + rc = strncpy_s(maxXLogFileName, MAXPGPATH, subDirEntry->d_name, strlen(subDirEntry->d_name) + 1); + securec_check(rc, "", ""); + maxXLogFileName[strlen(subDirEntry->d_name)] = '\0'; + } + } + (void)closedir(subDir); + } + } + (void)closedir(dir); +} + +XLogRecPtr SSFindMaxLSN(char *workingPath, char *returnMsg, int msgLen, pg_crc32 *maxLsnCrc, char* dssDirStr) +{ + struct dirent *entry = NULL; + XLogReaderState *xlogReader = NULL; + XLogPageReadPrivate readPrivate = { + .datadir = NULL, + .tli = 0 + }; + XLogRecord *record = NULL; + TimeLineID tli = 0; + XLogRecPtr maxLsn = InvalidXLogRecPtr; + XLogRecPtr startLsn = InvalidXLogRecPtr; + XLogRecPtr curLsn = InvalidXLogRecPtr; + char maxXLogFileName[MAXPGPATH] = {0}; + char *errorMsg = NULL; + bool findValidXLogFile = false; + uint32 xlogReadLogid = -1; + uint32 xlogReadLogSeg = -1; + char dssXlogDirStr[MAXPGPATH]; + errno_t rc = EOK; + DIR* dssDir = NULL; + bool breakLoops = false; + + /* Ranking xlog from large to small */ + FindMaxXlogFileName(dssDirStr, maxXLogFileName); + + if (sscanf_s(maxXLogFileName, "%08X%08X%08X", &tli, &xlogReadLogid, &xlogReadLogSeg) != 3) { + rc = snprintf_s(returnMsg, XLOG_READER_MAX_MSGLENTH, XLOG_READER_MAX_MSGLENTH - 1, + "failed to translate name to xlog: %s\n", maxXLogFileName); +#ifndef FRONTEND + securec_check_ss(rc, "", ""); +#else + securec_check_ss_c(rc, "", ""); +#endif + return InvalidXLogRecPtr; + } + + /* Initializing the ReaderState */ + rc = memset_s(&readPrivate, sizeof(XLogPageReadPrivate), 0, sizeof(XLogPageReadPrivate)); + securec_check_c(rc, "\0", "\0"); + readPrivate.tli = tli; + readPrivate.datadir = workingPath; + xlogReader = XLogReaderAllocate(&SimpleXLogPageRead, &readPrivate); + if (xlogReader == NULL) { + rc = snprintf_s(returnMsg, XLOG_READER_MAX_MSGLENTH, XLOG_READER_MAX_MSGLENTH - 1, "reader allocate failed.\n"); +#ifndef FRONTEND + securec_check_ss(rc, "", ""); +#else + securec_check_ss_c(rc, "", ""); +#endif + return InvalidXLogRecPtr; + } + + /* Start to find the max lsn from a valid xlogfile */ + startLsn = (xlogReadLogSeg * XLogSegSize) + ((XLogRecPtr)xlogReadLogid * XLogSegmentsPerXLogId * XLogSegSize); + while (!XLogRecPtrIsInvalid(startLsn) && !breakLoops) { + dssDir = opendir(dssDirStr); + /* find the first valid record from the bigger xlogrecord. then break */ + while (dssDir != NULL && (entry = readdir(dssDir)) != NULL) { + if (strncmp(entry->d_name, "pg_xlog", strlen("pg_xlog")) == 0) { + rc = snprintf_s(dssXlogDirStr, MAXPGPATH, MAXPGPATH - 1, "%s/%s", dssDirStr, entry->d_name); +#ifndef FRONTEND + securec_check_ss(rc, "", ""); +#else + securec_check_ss_c(rc, "", ""); +#endif + curLsn = XLogFindNextRecord(xlogReader, startLsn, NULL, dssXlogDirStr); + if (XLogRecPtrIsInvalid(curLsn)) { + CLOSE_FD(xlogreadfd); + } else { + findValidXLogFile = true; + breakLoops = true; + break; + } + } + } + startLsn = startLsn - XLogSegSize; + (void)closedir(dssDir); + } + + CLOSE_FD(xlogreadfd); + if (!findValidXLogFile) { + rc = snprintf_s(returnMsg, XLOG_READER_MAX_MSGLENTH, XLOG_READER_MAX_MSGLENTH - 1, + "no valid record in pg_xlog.\n"); +#ifndef FRONTEND + securec_check_ss(rc, "", ""); +#else + securec_check_ss_c(rc, "", ""); +#endif + /* Free all opened resources */ + if (xlogReader != NULL) { + XLogReaderFree(xlogReader); + xlogReader = NULL; + } + return InvalidXLogRecPtr; + } + + /* find the max lsn. */ + while(true) { + record = XLogReadRecordFromAllDir(dssDirStr, xlogReader, curLsn, &errorMsg); + if (record == NULL) { + break; + } + curLsn = InvalidXLogRecPtr; + *maxLsnCrc = record->xl_crc; + } + + maxLsn = xlogReader->ReadRecPtr; + if (XLogRecPtrIsInvalid(maxLsn)) { + rc = snprintf_s(returnMsg, XLOG_READER_MAX_MSGLENTH, XLOG_READER_MAX_MSGLENTH - 1, "%s", errorMsg); +#ifndef FRONTEND + securec_check_ss(rc, "", ""); +#else + securec_check_ss_c(rc, "", ""); +#endif + } else { + rc = snprintf_s(returnMsg, XLOG_READER_MAX_MSGLENTH, XLOG_READER_MAX_MSGLENTH - 1, + "find max lsn rec (%X/%X) success.\n", (uint32)(maxLsn >> 32), (uint32)maxLsn); +#ifndef FRONTEND + securec_check_ss(rc, "", ""); +#else + securec_check_ss_c(rc, "", ""); +#endif + } + + /* Free all opened resources */ + if (xlogReader != NULL) { + XLogReaderFree(xlogReader); + xlogReader = NULL; + } + + CLOSE_FD(xlogreadfd); + return maxLsn; +} + XLogRecPtr FindMaxLSN(char *workingPath, char *returnMsg, int msgLen, pg_crc32 *maxLsnCrc, uint32 *maxLsnLen, TimeLineID *returnTli, char* xlog_path) { diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index df3309928..bbd4ded83 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -42,6 +42,7 @@ extern void XLogReaderFree(XLogReaderState* state); /* Read the next XLog record. Returns NULL on end-of-WAL or failure */ extern struct XLogRecord* XLogReadRecord( XLogReaderState* state, XLogRecPtr recptr, char** errormsg, bool doDecode = true, char* xlog_path = NULL); +extern struct XLogRecord* XLogReadRecordFromAllDir(char* dirPath, XLogReaderState *xlogReader, XLogRecPtr curLsn, char** errorMsg); extern bool XLogRecGetBlockTag(XLogReaderState *record, uint8 block_id, RelFileNode *rnode, ForkNumber *forknum, BlockNumber *blknum, XLogPhyBlock *pblk = NULL); @@ -56,6 +57,7 @@ extern void XLogRecGetVMPhysicalBlock(const XLogReaderState *record, uint8 block extern void XLogReaderInvalReadState(XLogReaderState* state); extern XLogRecPtr XLogFindNextRecord(XLogReaderState* state, XLogRecPtr RecPtr, XLogRecPtr *endPtr = NULL, char* xlog_path = NULL); +extern XLogRecPtr SSFindMaxLSN(char *workingPath, char *returnMsg, int msgLen, pg_crc32 *maxLsnCrc, char* dssDirStr); extern XLogRecPtr FindMaxLSN(char* workingpath, char* returnmsg, int msg_len, pg_crc32* maxLsnCrc, uint32 *maxLsnLen = NULL, TimeLineID *returnTli = NULL, char* xlog_path = NULL); extern XLogRecPtr FindMinLSN(char *workingPath, char *returnMsg, int msgLen, pg_crc32 *minLsnCrc); From 6bc1089268517a783afdd9f13d1cdae35dfd21bc Mon Sep 17 00:00:00 2001 From: justbk <249396768@qq.com> Date: Fri, 1 Sep 2023 18:09:21 +0800 Subject: [PATCH 191/304] avoid u_sess is null in reform --- src/gausskernel/process/postmaster/og_record_time.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/gausskernel/process/postmaster/og_record_time.cpp b/src/gausskernel/process/postmaster/og_record_time.cpp index d615aa10d..041f59554 100644 --- a/src/gausskernel/process/postmaster/og_record_time.cpp +++ b/src/gausskernel/process/postmaster/og_record_time.cpp @@ -373,6 +373,10 @@ void OgRecordOperator::update_record_id() bool OgRecordOperator::report_enable() const { + if (u_sess == NULL) { + return false; + } + OgRecordStat* record_stat = (OgRecordStat*)get_record_cxt()->og_record_stat; if (record_stat == NULL) { return false; From 7095bb417b7fa85623044e974fa089ea8c01b4e0 Mon Sep 17 00:00:00 2001 From: chenxiaobin19 <1025221611@qq.com> Date: Sat, 2 Sep 2023 11:14:41 +0800 Subject: [PATCH 192/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=9F=BA=E7=A1=80?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=A4=8D=E5=88=B6=E7=9A=84=E5=A4=8D=E5=88=B6?= =?UTF-8?q?=E6=A7=BD=E6=AE=8B=E7=95=99=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../optimizer/commands/subscriptioncmds.cpp | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/gausskernel/optimizer/commands/subscriptioncmds.cpp b/src/gausskernel/optimizer/commands/subscriptioncmds.cpp index e4da27801..c3d21f758 100644 --- a/src/gausskernel/optimizer/commands/subscriptioncmds.cpp +++ b/src/gausskernel/optimizer/commands/subscriptioncmds.cpp @@ -1173,13 +1173,13 @@ void DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel) * the apply and tablesync workers and they can't restart because of * exclusive lock on the subscription. */ - rstates = GetSubscriptionRelations(subid, true); + rstates = GetSubscriptionRelations(subid, false); foreach (lc, rstates) { SubscriptionRelState *rstate = (SubscriptionRelState *)lfirst(lc); Oid relid = rstate->relid; /* Only cleanup resources of tablesync workers */ - if (!OidIsValid(relid)) + if (!OidIsValid(relid) || rstate->state == SUBREL_STATE_READY) continue; /* @@ -1236,8 +1236,14 @@ void DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel) /* * Drop the tablesync slots associated with removed tables. * - * For SYNCDONE/READY states, the tablesync slot is known to have - * already been dropped by the tablesync worker. + * For SYNCDONE state, the slot may remain. In the normal + * process(process_syncing_tables_for_sync), the slot will be + * deleted after the SYNCDONE setting, so if the DropSubscription + * is executed after the SYNCDONE and before the slot is deleted, + * DropSubscription needs to clean up the remaining slot. + * + * For READY state, the slot may also remain after system crashed + * and reovers. * * For other states, there is no certainty, maybe the slot does * not exist yet. Also, if we fail after removing some of the @@ -1245,12 +1251,10 @@ void DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel) * slots and fail. For these reasons, we allow missing_ok = true * for the drop. */ - if (rstate->state != SUBREL_STATE_SYNCDONE) { - char syncslotname[NAMEDATALEN] = {0}; + char syncslotname[NAMEDATALEN] = {0}; - ReplicationSlotNameForTablesync(subid, relid, syncslotname, sizeof(syncslotname)); - ReplicationSlotDropAtPubNode(syncslotname, true); - } + ReplicationSlotNameForTablesync(subid, relid, syncslotname, sizeof(syncslotname)); + ReplicationSlotDropAtPubNode(syncslotname, true); } list_free(rstates); @@ -1304,7 +1308,7 @@ void ReplicationSlotDropAtPubNode(char *slotname, bool missing_ok) walrcv_clear_result(res); FreeStringInfo(&cmd); - ereport(WARNING, (errmsg("replication slot \"%s\" does not exist on publisher", slotname))); + ereport(DEBUG2, (errmsg("replication slot \"%s\" does not exist on publisher", slotname))); return; } } From ae11b9a547651a0dbdc033a7aab672aa1eef6876 Mon Sep 17 00:00:00 2001 From: xiyanziran Date: Mon, 4 Sep 2023 02:41:05 +0000 Subject: [PATCH 193/304] update src/include/utils/guc.h. Signed-off-by: xiyanziran --- src/include/utils/guc.h | 56 ++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/src/include/utils/guc.h b/src/include/utils/guc.h index 2b1b1ea08..acd92a165 100755 --- a/src/include/utils/guc.h +++ b/src/include/utils/guc.h @@ -389,38 +389,38 @@ typedef enum { typedef enum { NO_REWRITE = 0, /* not allow lazy agg and magic set rewrite*/ - LAZY_AGG = (1<<0), /* allow lazy agg */ - MAGIC_SET = (1<<1), /* allow query qual push */ - PARTIAL_PUSH = (1<<2), /* allow partial push */ - SUBLINK_PULLUP_WITH_UNIQUE_CHECK = (1<<3), /* allow pull sublink with unqiue check */ - SUBLINK_PULLUP_DISABLE_REPLICATED = (1<<4), /* disable pull up sublink with replicated table */ - SUBLINK_PULLUP_IN_TARGETLIST = (1<<5), /* allow pull sublink in targetlist */ - PRED_PUSH = (1<<6), /* push predicate into subquery block */ - PRED_PUSH_NORMAL = (1<<7), - PRED_PUSH_FORCE = (1<<8), - SUBLINK_PULLUP_DISABLE_EXPR = (1<<9), /* disable pull sublink in expr clause */ - SUBLINK_PULLUP_ENHANCED = (1<<10) + LAZY_AGG = (1 << 0), /* allow lazy agg */ + MAGIC_SET = (1 << 1), /* allow query qual push */ + PARTIAL_PUSH = (1 << 2), /* allow partial push */ + SUBLINK_PULLUP_WITH_UNIQUE_CHECK = (1 << 3), /* allow pull sublink with unqiue check */ + SUBLINK_PULLUP_DISABLE_REPLICATED = (1 << 4), /* disable pull up sublink with replicated table */ + SUBLINK_PULLUP_IN_TARGETLIST = (1 << 5), /* allow pull sublink in targetlist */ + PRED_PUSH = (1 << 6), /* push predicate into subquery block */ + PRED_PUSH_NORMAL = (1 << 7), + PRED_PUSH_FORCE = (1 << 8), + SUBLINK_PULLUP_DISABLE_EXPR = (1 << 9), /* disable pull sublink in expr clause */ + SUBLINK_PULLUP_ENHANCED = (1 << 10) } rewrite_param; typedef enum { NO_BETA_FEATURE = 0, - SEL_SEMI_POISSON = (1<<0), /* use poisson distribution model to calibrate semi join selectivity */ - SEL_EXPR_INSTR = (1<<1), /* use pattern sel to calibrate instr() related base rel selectivity */ - PARAM_PATH_GEN = (1<<2), /* Parametrized Path Generation */ - RAND_COST_OPT = (1<<3), /* Optimizing sc_random_page_cost */ - PARAM_PATH_OPT = (1<<4), /* Parametrized Path Optimization. */ - PAGE_EST_OPT = (1<<5), /* More accurate (rowstored) index pages estimation */ - NO_UNIQUE_INDEX_FIRST = (1<<6), /* use unique index first rule in path generation */ - JOIN_SEL_WITH_CAST_FUNC = (1<<7), /* support cast function while calculating join selectivity */ - CANONICAL_PATHKEY = (1<<8), /* Use canonicalize pathkeys directly */ - INDEX_COST_WITH_LEAF_PAGES_ONLY = (1<<9), /* compute index cost with consideration of leaf-pages-only */ - PARTITION_OPFUSION = (1<<10), /* Enable partition opfusion */ - A_STYLE_COERCE = (1<<11), - PLPGSQL_STREAM_FETCHALL = (1<<12), /* fetch all tuple when has stream sql under plpgsql's for-loop */ - PREDPUSH_SAME_LEVEL = (1<<13), /* predpush same level */ - PARTITION_FDW_ON = (1<<14), /* support create foreign table on partitioned table */ - DISABLE_BITMAP_COST_WITH_LOSSY_PAGES = (1<<15), /* stop computing bitmap path cost with lossy pages */ - EXTRACT_PUSHDOWN_OR_CLAUSE = (1<<16) /* Extract restriction OR clauses. */ + SEL_SEMI_POISSON = (1 << 0), /* use poisson distribution model to calibrate semi join selectivity */ + SEL_EXPR_INSTR = (1 << 1), /* use pattern sel to calibrate instr() related base rel selectivity */ + PARAM_PATH_GEN = (1 << 2), /* Parametrized Path Generation */ + RAND_COST_OPT = (1 << 3), /* Optimizing sc_random_page_cost */ + PARAM_PATH_OPT = (1 << 4), /* Parametrized Path Optimization. */ + PAGE_EST_OPT = (1 << 5), /* More accurate (rowstored) index pages estimation */ + NO_UNIQUE_INDEX_FIRST = (1 << 6), /* use unique index first rule in path generation */ + JOIN_SEL_WITH_CAST_FUNC = (1 << 7), /* support cast function while calculating join selectivity */ + CANONICAL_PATHKEY = (1 << 8), /* Use canonicalize pathkeys directly */ + INDEX_COST_WITH_LEAF_PAGES_ONLY = (1 << 9), /* compute index cost with consideration of leaf-pages-only */ + PARTITION_OPFUSION = (1 << 10), /* Enable partition opfusion */ + A_STYLE_COERCE = (1 << 11), + PLPGSQL_STREAM_FETCHALL = (1 << 12), /* fetch all tuple when has stream sql under plpgsql's for-loop */ + PREDPUSH_SAME_LEVEL = (1 << 13), /* predpush same level */ + PARTITION_FDW_ON = (1 << 14), /* support create foreign table on partitioned table */ + DISABLE_BITMAP_COST_WITH_LOSSY_PAGES = (1 << 15), /* stop computing bitmap path cost with lossy pages */ + EXTRACT_PUSHDOWN_OR_CLAUSE = (1 << 16) /* Extract restriction OR clauses. */ } sql_beta_param; typedef enum { From a8ff9186079b9ad4e50f8686a05e9286686057eb Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Mon, 4 Sep 2023 14:06:40 +0800 Subject: [PATCH 194/304] =?UTF-8?q?9.4dss=E6=8E=A8=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/ddes/ddes_commit_id | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gausskernel/ddes/ddes_commit_id b/src/gausskernel/ddes/ddes_commit_id index e88d16fc9..1daea3912 100644 --- a/src/gausskernel/ddes/ddes_commit_id +++ b/src/gausskernel/ddes/ddes_commit_id @@ -1,3 +1,3 @@ dms_commit_id=d3b22f7482078513b98dcb3241e17b26962e8a36 -dss_commit_id=e8fbe469e1b320c34684ba7c15c5fbc5d5208cf8 +dss_commit_id=412bd233f74ffb84d2f9a715fd6d30450f56ff4c cbb_commit_id=41cadda4643656bcd856bd37d2b6908e44a5d46c \ No newline at end of file From 092db5c77ec3a662ba5e327e093b113c078ffa5c Mon Sep 17 00:00:00 2001 From: yunlongwang Date: Tue, 29 Aug 2023 17:58:25 +0800 Subject: [PATCH 195/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E5=9B=9E=E6=94=BE=E8=BF=87=E7=A8=8B=E4=B8=AD=20lsn=20=E6=A0=A1?= =?UTF-8?q?=E9=AA=8C=E5=A4=B1=E8=B4=A5=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../access/transam/parallel_recovery/dispatcher.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/gausskernel/storage/access/transam/parallel_recovery/dispatcher.cpp b/src/gausskernel/storage/access/transam/parallel_recovery/dispatcher.cpp index 7c5af05b5..91c8f84b1 100755 --- a/src/gausskernel/storage/access/transam/parallel_recovery/dispatcher.cpp +++ b/src/gausskernel/storage/access/transam/parallel_recovery/dispatcher.cpp @@ -651,8 +651,12 @@ void DispatchRedoRecordToFile(XLogReaderState *record, List *expectedTLIs, Times g_instance.attr.attr_storage.parallel_recovery_batch : 1; if (isNeedFullSync) ProcessPendingRecords(true); - else if (++g_dispatcher->pendingCount >= dispatch_batch || timeoutForDispatch()) + else if (++g_dispatcher->pendingCount >= dispatch_batch || timeoutForDispatch()) { ProcessPendingRecords(); + if ((g_dispatcher->dispatchEndRecPtr - g_dispatcher->dispatchFix.lastCheckLsn) > DISPATCH_FIX_SIZE) { + CheckDispatchCount(g_dispatcher->dispatchEndRecPtr); + } + } if (fatalerror == true) { /* output panic error info */ @@ -661,10 +665,6 @@ void DispatchRedoRecordToFile(XLogReaderState *record, List *expectedTLIs, Times errmsg("[REDO_LOG_TRACE]DispatchRedoRecord encounter fatal error:rmgrID:%u, info:%u, indexid:%u", rmid, (uint32)XLogRecGetInfo(record), indexid))); } - - if ((g_dispatcher->dispatchEndRecPtr - g_dispatcher->dispatchFix.lastCheckLsn) > DISPATCH_FIX_SIZE) { - CheckDispatchCount(g_dispatcher->dispatchEndRecPtr); - } } else { ereport(PANIC, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), errmsg("[REDO_LOG_TRACE]DispatchRedoRecord could not be here config recovery num %d, work num %u", From 60df5b072d51c7ad419c39452695d25d8a88f524 Mon Sep 17 00:00:00 2001 From: bowenliu Date: Mon, 4 Sep 2023 20:02:12 +0800 Subject: [PATCH 196/304] fix ss txnstatus cache size guc --- src/bin/gs_guc/cluster_guc.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/src/bin/gs_guc/cluster_guc.conf b/src/bin/gs_guc/cluster_guc.conf index 321353905..c58764948 100755 --- a/src/bin/gs_guc/cluster_guc.conf +++ b/src/bin/gs_guc/cluster_guc.conf @@ -754,6 +754,7 @@ light_comm|bool|0,0|NULL|NULL| ignore_standby_lsn_window|int|0,2147483647|ms|NULL| ignore_feedback_xmin_window|int|0,2147483647|ms|NULL| ss_enable_bcast_snapshot|bool|0,0|NULL|NULL| +ss_txnstatus_cache_size|int|0,524288|NULL|NULL| subscription_conflict_resolution|enum|error,apply_remote,keep_local|NULL|NULL| time_record_level|int|0,10|NULL|NULL| [cmserver] From eb2a65822c99ee4cabf4d6fd3c6a12ccd181a27a Mon Sep 17 00:00:00 2001 From: teooooozhang Date: Mon, 27 Feb 2023 16:23:32 +0800 Subject: [PATCH 197/304] =?UTF-8?q?issue=E4=BF=AE=E5=A4=8D=EF=BC=9Ags=5Fba?= =?UTF-8?q?sebackup=E5=91=BD=E4=BB=A4=E5=86=99=E5=85=A5stdout=E6=89=A7?= =?UTF-8?q?=E8=A1=8C=E5=A4=B1=E8=B4=A5=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/pg_basebackup/pg_basebackup.cpp | 29 +++++++++++++++++++++---- src/bin/pg_ctl/fetchmot.cpp | 20 +++++++++++++++-- 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/src/bin/pg_basebackup/pg_basebackup.cpp b/src/bin/pg_basebackup/pg_basebackup.cpp index 05beb0e98..66540c0a9 100644 --- a/src/bin/pg_basebackup/pg_basebackup.cpp +++ b/src/bin/pg_basebackup/pg_basebackup.cpp @@ -635,8 +635,6 @@ static void ReceiveTarFile(PGconn *conn, PGresult *res, int rownum) duplicatedfd = -1; disconnect_and_exit(1); } - close(duplicatedfd); - duplicatedfd = -1; } else #endif tarfile = stdout; @@ -692,6 +690,10 @@ static void ReceiveTarFile(PGconn *conn, PGresult *res, int rownum) /* Compression is in use */ pg_log(stderr, _("%s: could not create compressed file \"%s\": %s\n"), progname, filename, get_gz_error(ztarfile)); + if(duplicatedfd != -1) { + close(duplicatedfd); + duplicatedfd = -1; + } disconnect_and_exit(1); } } else @@ -734,6 +736,10 @@ static void ReceiveTarFile(PGconn *conn, PGresult *res, int rownum) progname, filename, get_gz_error(ztarfile)); + if (duplicatedfd != -1) { + close(duplicatedfd); + duplicatedfd = -1; + } disconnect_and_exit(1); } } else @@ -764,6 +770,10 @@ static void ReceiveTarFile(PGconn *conn, PGresult *res, int rownum) progname, filename, get_gz_error(ztarfile)); + if (duplicatedfd != -1) { + close(duplicatedfd); + duplicatedfd = -1; + } disconnect_and_exit(1); } } else @@ -779,6 +789,12 @@ static void ReceiveTarFile(PGconn *conn, PGresult *res, int rownum) progress_report(rownum, filename); } /* while (1) */ +#ifdef HAVE_LIBZ + if (duplicatedfd != -1) { + close(duplicatedfd); + duplicatedfd = -1; + } +#endif if (copybuf != NULL) { PQfreemem(copybuf); copybuf = NULL; @@ -1985,11 +2001,16 @@ static int GsBaseBackup(int argc, char** argv) GS_FREE(basedir); check_env_value_c(optarg); char realDir[PATH_MAX] = {0}; - if (realpath(optarg, realDir) == nullptr) { + bool argIsMinus = (strcmp(optarg, "-") == 0); + if (!argIsMinus && realpath(optarg, realDir) == nullptr) { pg_log(stderr, _("%s: realpath dir \"%s\" failed: %m\n"), progname, optarg); exit(1); } - basedir = xstrdup(realDir); + if (argIsMinus) { + basedir = xstrdup(optarg); + } else { + basedir = xstrdup(realDir); + } break; } case 'F': diff --git a/src/bin/pg_ctl/fetchmot.cpp b/src/bin/pg_ctl/fetchmot.cpp index d1bb9ea00..c2be7c778 100644 --- a/src/bin/pg_ctl/fetchmot.cpp +++ b/src/bin/pg_ctl/fetchmot.cpp @@ -107,8 +107,6 @@ static void MotReceiveAndAppendTarFile(const char* basedir, const char* chkptNam duplicatedfd = -1; disconnect_and_exit(1); } - close(duplicatedfd); - duplicatedfd = -1; } else #endif tarfile = stdout; @@ -158,6 +156,10 @@ static void MotReceiveAndAppendTarFile(const char* basedir, const char* chkptNam if (gzclose(ztarfile) != 0) { fprintf(stderr, _("%s: could not close compressed file \"%s\": %s\n"), progname, filename, get_gz_error(ztarfile)); + if (duplicatedfd != -1) { + close(duplicatedfd); + duplicatedfd = -1; + } disconnect_and_exit(1); } } else @@ -229,6 +231,10 @@ static void MotReceiveAndAppendTarFile(const char* basedir, const char* chkptNam if (ztarfile != NULL) { if (!writeGzFile(ztarfile, copybuf, r)) { fprintf(stderr, _("%s: could not write to file \"%s\": %s\n"), progname, filename, strerror(errno)); + if (duplicatedfd != -1) { + close(duplicatedfd); + duplicatedfd = -1; + } disconnect_and_exit(1); } } else @@ -271,6 +277,10 @@ static void MotReceiveAndAppendTarFile(const char* basedir, const char* chkptNam if (ztarfile != NULL) { if (!writeGzFile(ztarfile, copybuf, r)) { fprintf(stderr, _("%s: could not write to file \"%s\": %s\n"), progname, filename, strerror(errno)); + if (duplicatedfd != -1) { + close(duplicatedfd); + duplicatedfd = -1; + } disconnect_and_exit(1); } } else @@ -290,6 +300,12 @@ static void MotReceiveAndAppendTarFile(const char* basedir, const char* chkptNam } } /* continuing data in existing file */ } /* loop over all data blocks */ +#ifdef HAVE_LIBZ + if (duplicatedfd != -1) { + close(duplicatedfd); + duplicatedfd = -1; + } +#endif if (tarfile != NULL) { fclose(tarfile); tarfile = NULL; From 810f96544de25359103387a224e76df295b83ee2 Mon Sep 17 00:00:00 2001 From: Julong-Li <584147810@qq.com> Date: Mon, 4 Sep 2023 14:30:07 +0800 Subject: [PATCH 198/304] =?UTF-8?q?issue:=E8=A7=A3=E5=86=B3=E5=AD=98?= =?UTF-8?q?=E5=82=A8=E8=BF=87=E7=A8=8B=E5=88=9B=E5=BB=BA=E6=97=B6row=20typ?= =?UTF-8?q?e=E9=94=99=E8=AF=AF=E4=BD=BF=E7=94=A8=E6=97=B6=E4=B8=8D?= =?UTF-8?q?=E6=8A=A5=E9=94=99=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/pl/plpgsql/src/gram.y | 8 +++ .../expected/test_row_type_in_proc.out | 61 +++++++++++++++++++ src/test/regress/parallel_schedule0 | 4 +- src/test/regress/parallel_schedule0A | 2 +- .../regress/sql/test_row_type_in_proc.sql | 38 ++++++++++++ 5 files changed, 110 insertions(+), 3 deletions(-) create mode 100644 src/test/regress/expected/test_row_type_in_proc.out create mode 100755 src/test/regress/sql/test_row_type_in_proc.sql diff --git a/src/common/pl/plpgsql/src/gram.y b/src/common/pl/plpgsql/src/gram.y index f87a40837..64d200c1f 100755 --- a/src/common/pl/plpgsql/src/gram.y +++ b/src/common/pl/plpgsql/src/gram.y @@ -11282,6 +11282,14 @@ read_into_target(PLpgSQL_rec **rec, PLpgSQL_row **row, bool *strict, int firstto errmsg("record or row variable cannot be part of multiple-item INTO list"), parser_errposition(yylloc))); } + if (tok == '.') { + const char* message = "Improper use of '.*'. The '.*' operator cannot be used with a row type variable."; + InsertErrorMessage(message, plpgsql_yylloc); + ereport(errstate, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("Improper use of '.*'. The '.*' operator cannot be used with a row type variable."), + parser_errposition(yylloc))); + } if (tok == T_DATUM || tok == T_VARRAY_VAR || tok == T_TABLE_VAR || tok == T_PACKAGE_VARIABLE) { const char* message = "syntax error, expected \",\""; diff --git a/src/test/regress/expected/test_row_type_in_proc.out b/src/test/regress/expected/test_row_type_in_proc.out new file mode 100644 index 000000000..08919c6ae --- /dev/null +++ b/src/test/regress/expected/test_row_type_in_proc.out @@ -0,0 +1,61 @@ +create table type_test(a int, b char(10)); +create table test_row_type(a int, b char(10)); +create or replace procedure test_row_type_in_pro() +as declare +rec type_test%rowtype; +a int; +begin +select x.* , x2.a into rec, a from test_row_type(); +end; +/ +ERROR: record or row variable cannot be part of multiple-item INTO list +LINE 5: select x.* , x2.a into rec, a from test_row_type(); + ^ +QUERY: declare +rec type_test%rowtype; +a int; +begin +select x.* , x2.a into rec, a from test_row_type(); +end +create or replace procedure test_row_type_in_pro() +as declare +rec type_test%rowtype; +a int; +begin +select x.* , x2.a into rec.*, a from test_row_type(); +end; +/ +ERROR: Improper use of '.*'. The '.*' operator cannot be used with a row type variable. +LINE 5: select x.* , x2.a into rec.*, a from test_row_type(); + ^ +QUERY: declare +rec type_test%rowtype; +a int; +begin +select x.* , x2.a into rec.*, a from test_row_type(); +end +create or replace procedure test_row_type_in_pro() +as declare +rec type_test%rowtype; +a int; +begin +select x.* into rec.* from test_row_type(); +end; +/ +ERROR: Improper use of '.*'. The '.*' operator cannot be used with a row type variable. +LINE 5: select x.* into rec.* from test_row_type(); + ^ +QUERY: declare +rec type_test%rowtype; +a int; +begin +select x.* into rec.* from test_row_type(); +end +create or replace procedure test_row_type_in_pro() +as declare +rec type_test%rowtype; +a int; +begin +select x.* into rec from test_row_type(); +end; +/ diff --git a/src/test/regress/parallel_schedule0 b/src/test/regress/parallel_schedule0 index 90cdb60ca..6635ec720 100644 --- a/src/test/regress/parallel_schedule0 +++ b/src/test/regress/parallel_schedule0 @@ -32,7 +32,7 @@ test: replace_func_with_two_args trunc_func_for_date nlssort_pinyin updatable_vi # test multiple statistics test: functional_dependency -test: pg_proc_test +test: pg_proc_test test_row_type_in_proc # test fdw # NOTICE: In the "fdw_prepare", we copy the fdw test to be used from contrib into regress sql set. @@ -1099,4 +1099,4 @@ test: enable_expr_fusion_flatten # test for on update timestamp and generated column test: on_update_session1 on_update_session2 -test: ts_gb18030_utf8 \ No newline at end of file +test: ts_gb18030_utf8 diff --git a/src/test/regress/parallel_schedule0A b/src/test/regress/parallel_schedule0A index 7ef668d0b..547f93ead 100644 --- a/src/test/regress/parallel_schedule0A +++ b/src/test/regress/parallel_schedule0A @@ -29,7 +29,7 @@ test: replace_func_with_two_args trunc_func_for_date nlssort_pinyin updatable_vi # test multiple statistics test: functional_dependency -test: pg_proc_test +test: pg_proc_test test_row_type_in_proc # test fdw # NOTICE: In the "fdw_prepare", we copy the fdw test to be used from contrib into regress sql set. diff --git a/src/test/regress/sql/test_row_type_in_proc.sql b/src/test/regress/sql/test_row_type_in_proc.sql new file mode 100755 index 000000000..dbfb0808e --- /dev/null +++ b/src/test/regress/sql/test_row_type_in_proc.sql @@ -0,0 +1,38 @@ +create table type_test(a int, b char(10)); +create table test_row_type(a int, b char(10)); + +create or replace procedure test_row_type_in_pro() +as declare +rec type_test%rowtype; +a int; +begin +select x.* , x2.a into rec, a from test_row_type(); +end; +/ + +create or replace procedure test_row_type_in_pro() +as declare +rec type_test%rowtype; +a int; +begin +select x.* , x2.a into rec.*, a from test_row_type(); +end; +/ + +create or replace procedure test_row_type_in_pro() +as declare +rec type_test%rowtype; +a int; +begin +select x.* into rec.* from test_row_type(); +end; +/ + +create or replace procedure test_row_type_in_pro() +as declare +rec type_test%rowtype; +a int; +begin +select x.* into rec from test_row_type(); +end; +/ From 7b16e5499675cbe1f131104ed437e47afdeb6084 Mon Sep 17 00:00:00 2001 From: dongning12 Date: Tue, 5 Sep 2023 16:25:46 +0800 Subject: [PATCH 199/304] =?UTF-8?q?=E3=80=90=E8=B5=84=E6=BA=90=E6=B1=A0?= =?UTF-8?q?=E5=8C=96=E3=80=91=E9=80=82=E9=85=8DDMS=E4=BE=A7=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0smon=5Frecycle=E7=BA=BF=E7=A8=8B=EF=BC=9BDMS=E6=8E=A8?= =?UTF-8?q?=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/ddes/ddes_commit_id | 4 ++-- src/include/storage/proc.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/gausskernel/ddes/ddes_commit_id b/src/gausskernel/ddes/ddes_commit_id index 1daea3912..d22913055 100644 --- a/src/gausskernel/ddes/ddes_commit_id +++ b/src/gausskernel/ddes/ddes_commit_id @@ -1,3 +1,3 @@ -dms_commit_id=d3b22f7482078513b98dcb3241e17b26962e8a36 +dms_commit_id=85eaf1d9de8d7486fdfa7a154405701bf294f490 dss_commit_id=412bd233f74ffb84d2f9a715fd6d30450f56ff4c -cbb_commit_id=41cadda4643656bcd856bd37d2b6908e44a5d46c \ No newline at end of file +cbb_commit_id=7a7e77f3dec94b6b958bea12c97b92d21dfa0bb5 \ No newline at end of file diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h index b1abe2d8b..6b25fa8ce 100755 --- a/src/include/storage/proc.h +++ b/src/include/storage/proc.h @@ -466,7 +466,7 @@ const int MAX_COMPACTION_THREAD_NUM = 10; #define NUM_DMS_REFORM_CALLLBACK_PROCS (5) #define NUM_DMS_LSNR_CALLBACK_PROC (1) -#define NUM_DMS_SMON_CALLBACK_PROC (1) +#define NUM_DMS_SMON_CALLBACK_PROC (2) // smon + smon_recycle #define NUM_DMS_PARALLEL_CALLBACK_PROC (g_instance.attr.attr_storage.dms_attr.parallel_thread_num <= 1 ? 0 : \ g_instance.attr.attr_storage.dms_attr.parallel_thread_num) #define NUM_DMS_RDMA_THREAD_CNT (g_instance.attr.attr_storage.dms_attr.work_thread_count * 2) From ee14277d769a58a8997ff9b36ebe661b33d20bd2 Mon Sep 17 00:00:00 2001 From: he-shaoyu Date: Tue, 29 Aug 2023 16:04:29 +0800 Subject: [PATCH 200/304] =?UTF-8?q?=E6=B7=BB=E5=8A=A0create=20if=20not=20e?= =?UTF-8?q?xists=E8=AF=AD=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- doc/src/sgml/ref/create_sequence.sgmlin | 2 +- src/common/backend/nodes/copyfuncs.cpp | 1 + src/common/backend/nodes/equalfuncs.cpp | 1 + src/common/backend/parser/gram.y | 28 +++++++++++++++++++ .../optimizer/commands/sequence/sequence.cpp | 13 ++++++++- src/include/nodes/parsenodes.h | 1 + .../test_create_sequence_if_not_exists.out | 14 ++++++++++ src/test/regress/parallel_schedule0 | 2 +- src/test/regress/parallel_schedule0A | 2 +- .../test_create_sequence_if_not_exists.sql | 12 ++++++++ 10 files changed, 72 insertions(+), 4 deletions(-) create mode 100644 src/test/regress/expected/test_create_sequence_if_not_exists.out create mode 100644 src/test/regress/sql/test_create_sequence_if_not_exists.sql diff --git a/doc/src/sgml/ref/create_sequence.sgmlin b/doc/src/sgml/ref/create_sequence.sgmlin index 8ef9fd308..7520fd49b 100644 --- a/doc/src/sgml/ref/create_sequence.sgmlin +++ b/doc/src/sgml/ref/create_sequence.sgmlin @@ -16,7 +16,7 @@ -CREATE [ LARGE ] SEQUENCE name [ INCREMENT [ BY ] increment ] +CREATE [ LARGE ] SEQUENCE [ IF NOT EXISTS ] name [ INCREMENT [ BY ] increment ] [ MINVALUE minvalue | NO MINVALUE | NOMINVALUE] [ MAXVALUE maxvalue | NO MAXVALUE | NOMAXVALUE] [ START [ WITH ] start ] [ CACHE cache ] [ [ NO ] CYCLE | NOCYCLE] [ OWNED BY { table_name.column_name | NONE } ]; diff --git a/src/common/backend/nodes/copyfuncs.cpp b/src/common/backend/nodes/copyfuncs.cpp index 018742ef0..2340f2cda 100644 --- a/src/common/backend/nodes/copyfuncs.cpp +++ b/src/common/backend/nodes/copyfuncs.cpp @@ -5901,6 +5901,7 @@ static CreateSeqStmt* _copyCreateSeqStmt(const CreateSeqStmt* from) COPY_SCALAR_FIELD(uuid); COPY_SCALAR_FIELD(canCreateTempSeq); COPY_SCALAR_FIELD(is_large); + COPY_SCALAR_FIELD(missing_ok); return newnode; } diff --git a/src/common/backend/nodes/equalfuncs.cpp b/src/common/backend/nodes/equalfuncs.cpp index 0395e0e99..f6264d368 100644 --- a/src/common/backend/nodes/equalfuncs.cpp +++ b/src/common/backend/nodes/equalfuncs.cpp @@ -1893,6 +1893,7 @@ static bool _equalCreateSeqStmt(const CreateSeqStmt* a, const CreateSeqStmt* b) COMPARE_SCALAR_FIELD(uuid); COMPARE_SCALAR_FIELD(canCreateTempSeq); COMPARE_SCALAR_FIELD(is_large); + COMPARE_SCALAR_FIELD(missing_ok); return true; } diff --git a/src/common/backend/parser/gram.y b/src/common/backend/parser/gram.y index 332202531..9cc50cd8f 100644 --- a/src/common/backend/parser/gram.y +++ b/src/common/backend/parser/gram.y @@ -9861,6 +9861,34 @@ CreateSeqStmt: n->sequence = $5; n->options = $6; + n->missing_ok = false; + n->ownerId = InvalidOid; +/* PGXC_BEGIN */ + n->is_serial = false; +/* PGXC_END */ + n->uuid = 0; + n->canCreateTempSeq = false; + $$ = (Node *)n; + } + | CREATE OptTemp opt_large_seq SEQUENCE IF_P NOT EXISTS qualified_name OptSeqOptList + { + CreateSeqStmt *n = makeNode(CreateSeqStmt); + $8->relpersistence = $2; + n->is_large = $3; +#ifdef ENABLE_MULTIPLE_NODES + if (n->is_large) { + const char* message = "large sequence is not supported."; + InsertErrorMessage(message, u_sess->plsql_cxt.plpgsql_yylloc); + ereport(ERROR, + (errmodule(MOD_PARSER), + errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("large sequence is not supported."))); + } +#endif + + n->sequence = $8; + n->options = $9; + n->missing_ok = true; n->ownerId = InvalidOid; /* PGXC_BEGIN */ n->is_serial = false; diff --git a/src/gausskernel/optimizer/commands/sequence/sequence.cpp b/src/gausskernel/optimizer/commands/sequence/sequence.cpp index 3255a854c..8dd21db7c 100644 --- a/src/gausskernel/optimizer/commands/sequence/sequence.cpp +++ b/src/gausskernel/optimizer/commands/sequence/sequence.cpp @@ -821,6 +821,18 @@ static ObjectAddress DefineSequence(CreateSeqStmt* seq) bool isUseLocalSeq = false; Oid namespaceOid = InvalidOid; ObjectAddress address; + Oid existing_relid = InvalidOid; + Oid namespaceId = InvalidOid; + char rel_kind = large ? RELKIND_LARGE_SEQUENCE : RELKIND_SEQUENCE; + + if (seq->missing_ok) { + namespaceId = RangeVarGetAndCheckCreationNamespace(seq->sequence, NoLock, &existing_relid, rel_kind); + if (existing_relid != InvalidOid) { + char* namespace_of_existing_rel = get_namespace_name(namespaceId); + ereport(NOTICE, (errmodule(MOD_COMMAND), errmsg("relation \"%s\" already exists in schema \"%s\", skipping", seq->sequence->relname, namespace_of_existing_rel))); + return InvalidObjectAddress; + } + } #ifdef PGXC /* PGXC_COORD */ GTM_Sequence start_value = 1; @@ -996,7 +1008,6 @@ static ObjectAddress DefineSequence(CreateSeqStmt* seq) stmt->tablespacename = NULL; stmt->if_not_exists = false; stmt->charset = PG_INVALID_ENCODING; - char rel_kind = large ? RELKIND_LARGE_SEQUENCE : RELKIND_SEQUENCE; address = DefineRelation(stmt, rel_kind, seq->ownerId, NULL); seqoid = address.objectId; Assert(seqoid != InvalidOid); diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index fe7e86ae9..d87df917a 100755 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -1131,6 +1131,7 @@ typedef struct CreateSeqStmt { int64 uuid; /* UUID of the sequence, mark unique sequence globally */ bool canCreateTempSeq; /* create sequence when "create table (like )" */ bool is_large; + bool missing_ok; /* skip error if a Sequence is exists */ } CreateSeqStmt; typedef struct AlterSeqStmt { diff --git a/src/test/regress/expected/test_create_sequence_if_not_exists.out b/src/test/regress/expected/test_create_sequence_if_not_exists.out new file mode 100644 index 000000000..a4cea9e39 --- /dev/null +++ b/src/test/regress/expected/test_create_sequence_if_not_exists.out @@ -0,0 +1,14 @@ +create schema create_sequence_if_not_exists; +set current_schema = create_sequence_if_not_exists; +create sequence seq start with 1; +create sequence seq start with 1; +ERROR: relation "seq" already exists in schema "create_sequence_if_not_exists" +DETAIL: creating new table with existing name in the same schema +create sequence if not exists seq start with 1; +NOTICE: relation "seq" already exists in schema "create_sequence_if_not_exists", skipping +drop sequence seq; +create sequence if not exists seq start with 1; +create sequence if not exists seq start with 2; +NOTICE: relation "seq" already exists in schema "create_sequence_if_not_exists", skipping +drop sequence seq; +drop schema create_sequence_if_not_exists; diff --git a/src/test/regress/parallel_schedule0 b/src/test/regress/parallel_schedule0 index d3135fa2d..0f387ca1f 100644 --- a/src/test/regress/parallel_schedule0 +++ b/src/test/regress/parallel_schedule0 @@ -639,7 +639,7 @@ test: copy_new_gram copy_gbk_test copy_gb18030_test # Another group of parallel tests # ---------- test: create_function_3 vacuum -test: drop_if_exists drop_database test_if_not_exists test_create_index_if_not_exists +test: drop_if_exists drop_database test_if_not_exists test_create_index_if_not_exists test_create_sequence_if_not_exists #test: constraints #test: errors subplan_base diff --git a/src/test/regress/parallel_schedule0A b/src/test/regress/parallel_schedule0A index 7ef668d0b..0259fd6ff 100644 --- a/src/test/regress/parallel_schedule0A +++ b/src/test/regress/parallel_schedule0A @@ -436,7 +436,7 @@ test: stable_function_shippable # Another group of parallel tests # ---------- test: create_function_3 vacuum -test: drop_if_exists drop_database test_if_not_exists test_create_index_if_not_exists +test: drop_if_exists drop_database test_if_not_exists test_create_index_if_not_exists test_create_sequence_if_not_exists #test: constraints #test: errors subplan_base diff --git a/src/test/regress/sql/test_create_sequence_if_not_exists.sql b/src/test/regress/sql/test_create_sequence_if_not_exists.sql new file mode 100644 index 000000000..42f7de9cd --- /dev/null +++ b/src/test/regress/sql/test_create_sequence_if_not_exists.sql @@ -0,0 +1,12 @@ +create schema create_sequence_if_not_exists; +set current_schema = create_sequence_if_not_exists; + +create sequence seq start with 1; +create sequence seq start with 1; + +create sequence if not exists seq start with 1; +drop sequence seq; +create sequence if not exists seq start with 1; +create sequence if not exists seq start with 2; +drop sequence seq; +drop schema create_sequence_if_not_exists; \ No newline at end of file From c7656f6957029bf013d24eda5c603f75eb6259fb Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Wed, 6 Sep 2023 14:54:26 +0800 Subject: [PATCH 201/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=8F=8C=E9=9B=86?= =?UTF-8?q?=E7=BE=A4system=20id=E5=AF=B9=E4=B8=8D=E4=B8=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/storage/replication/ss_cluster_replication.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/gausskernel/storage/replication/ss_cluster_replication.cpp b/src/gausskernel/storage/replication/ss_cluster_replication.cpp index a60d52255..b61e2dcc6 100644 --- a/src/gausskernel/storage/replication/ss_cluster_replication.cpp +++ b/src/gausskernel/storage/replication/ss_cluster_replication.cpp @@ -43,6 +43,8 @@ void WriteSSDoradoCtlInfoFile() SS_DORADO_CTRL_FILE))); } + ctlInfo->systemIdentifier = GetSystemIdentifier(); + pg_crc32c crc = CalShareStorageCtlInfoCrc(ctlInfo); ctlInfo->crc = crc; From 954c93a7d9372ab2307dcff7a0fc870ebff83fd3 Mon Sep 17 00:00:00 2001 From: dongning12 Date: Wed, 6 Sep 2023 15:39:46 +0800 Subject: [PATCH 202/304] =?UTF-8?q?[=E8=B5=84=E6=BA=90=E6=B1=A0=E5=8C=96]s?= =?UTF-8?q?napshot=20xmin=E7=9A=84hash=E8=A1=A8=E4=BD=BF=E7=94=A8=E5=A4=9A?= =?UTF-8?q?=E4=B8=AApartition=E9=94=81=E8=BF=9B=E8=A1=8C=E5=B9=B6=E5=8F=91?= =?UTF-8?q?=E6=8E=A7=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ddes/adapter/ss_dms_auxiliary.cpp | 27 +++++--- .../ddes/adapter/ss_dms_callback.cpp | 11 +++- src/gausskernel/ddes/adapter/ss_xmin.cpp | 61 ++++++++++++------- .../process/threadpool/knl_instance.cpp | 2 +- src/gausskernel/storage/lmgr/lwlock.cpp | 7 ++- src/include/ddes/dms/ss_xmin.h | 8 ++- src/include/storage/lock/lwlock.h | 8 ++- 7 files changed, 88 insertions(+), 36 deletions(-) diff --git a/src/gausskernel/ddes/adapter/ss_dms_auxiliary.cpp b/src/gausskernel/ddes/adapter/ss_dms_auxiliary.cpp index 7f387a9d1..2a054cf84 100644 --- a/src/gausskernel/ddes/adapter/ss_dms_auxiliary.cpp +++ b/src/gausskernel/ddes/adapter/ss_dms_auxiliary.cpp @@ -29,7 +29,9 @@ #include "storage/procarray.h" #include "storage/ipc.h" -#define DMS_AUXILIARY_SLEEP_TIME (1000) // 1s 1000ms +#define DMS_AUXILIARY_PRIMARY_SLEEP_TIME (5000) // 5s 5000ms +#define DMS_AUXILIARY_STANDBY_SLEEP_TIME (1000) // 1s 1000ms + static void dms_auxiliary_request_shutdown_handler(SIGNAL_ARGS) { int save_errno = errno; @@ -74,6 +76,12 @@ void dms_auxiliary_handle_exception() /* Prevent interrupts while cleaning up */ HOLD_INTERRUPTS(); + if (hash_get_seq_num() > 0) { + release_all_seq_scan(); + } + + LWLockReleaseAll(); + /* Report the error to the server log */ EmitErrorReport(); @@ -113,8 +121,10 @@ void DmsAuxiliaryMain(void) ctl.keysize = sizeof(ss_snap_xmin_key_t); ctl.entrysize = sizeof(ss_snap_xmin_item_t); ctl.hash = tag_hash; + ctl.num_partitions = NUM_SS_SNAPSHOT_XMIN_CACHE_PARTITIONS; ss_xmin_info_t *xmin_info = &g_instance.dms_cxt.SSXminInfo; - xmin_info->snap_cache = HeapMemInitHash("DMS snap xmin cache", 60, 30000, &ctl, HASH_ELEM | HASH_FUNCTION); + xmin_info->snap_cache = HeapMemInitHash("DMS snapshot xmin cache", 60, 30000, &ctl, + HASH_ELEM | HASH_FUNCTION | HASH_PARTITION); for (;;) { if (t_thrd.dms_aux_cxt.shutdown_requested) { @@ -129,13 +139,16 @@ void DmsAuxiliaryMain(void) if (SS_NORMAL_PRIMARY) { MaintXminInPrimary(); + int rc = WaitLatch(&t_thrd.proc->procLatch, WL_TIMEOUT | WL_POSTMASTER_DEATH, DMS_AUXILIARY_PRIMARY_SLEEP_TIME); + if (rc & WL_POSTMASTER_DEATH) { + gs_thread_exit(1); + } } else if (SS_NORMAL_STANDBY) { MaintXminInStandby(); - } - - int rc = WaitLatch(&t_thrd.proc->procLatch, WL_TIMEOUT | WL_POSTMASTER_DEATH, DMS_AUXILIARY_SLEEP_TIME); - if (rc & WL_POSTMASTER_DEATH) { - gs_thread_exit(1); + int rc = WaitLatch(&t_thrd.proc->procLatch, WL_TIMEOUT | WL_POSTMASTER_DEATH, DMS_AUXILIARY_STANDBY_SLEEP_TIME); + if (rc & WL_POSTMASTER_DEATH) { + gs_thread_exit(1); + } } } } \ No newline at end of file diff --git a/src/gausskernel/ddes/adapter/ss_dms_callback.cpp b/src/gausskernel/ddes/adapter/ss_dms_callback.cpp index 6cfa823f1..c0af89cf8 100644 --- a/src/gausskernel/ddes/adapter/ss_dms_callback.cpp +++ b/src/gausskernel/ddes/adapter/ss_dms_callback.cpp @@ -1693,10 +1693,17 @@ static void SSXminInfoPrepare() { ss_xmin_info_t *xmin_info = &g_instance.dms_cxt.SSXminInfo; if (g_instance.dms_cxt.SSReformInfo.dms_role == DMS_ROLE_REFORMER) { + SpinLockAcquire(&xmin_info->global_oldest_xmin_lock); + xmin_info->prev_global_oldest_xmin = xmin_info->global_oldest_xmin; xmin_info->global_oldest_xmin_active = false; + xmin_info->global_oldest_xmin = MaxTransactionId; + SpinLockRelease(&xmin_info->global_oldest_xmin_lock); for (int i = 0; i < DMS_MAX_INSTANCES; i++) { - xmin_info->node_table[i].active = false; - xmin_info->node_table[i].notify_oldest_xmin = MaxTransactionId; + ss_node_xmin_item_t *item = &xmin_info->node_table[i]; + SpinLockAcquire(&item->item_lock); + item->active = false; + item->notify_oldest_xmin = MaxTransactionId; + SpinLockRelease(&item->item_lock); } } xmin_info->bitmap_active_nodes = 0; diff --git a/src/gausskernel/ddes/adapter/ss_xmin.cpp b/src/gausskernel/ddes/adapter/ss_xmin.cpp index 5decb60ad..c5475c237 100644 --- a/src/gausskernel/ddes/adapter/ss_xmin.cpp +++ b/src/gausskernel/ddes/adapter/ss_xmin.cpp @@ -33,6 +33,11 @@ extern void CalculateLocalLatestSnapshot(bool forceCalc); +uint32 SSSnapshotXminKeyHashCode(const ss_snap_xmin_key_t *key) +{ + return get_hash_value(g_instance.dms_cxt.SSXminInfo.snap_cache, key); +} + uint64 GetOldestXminInNodeTable() { ss_xmin_info_t *xmin_info = &g_instance.dms_cxt.SSXminInfo; @@ -70,14 +75,18 @@ void MaintXminInPrimary(void) uint64 snap_xmin = MaxTransactionId; TimestampTz cur_time = GetCurrentTimestamp(); - while ((xmin_item = (ss_snap_xmin_item_t*)hash_seq_search(&hash_seq)) != NULL) { - if (SS_IN_REFORM) { - return; - } + for (int i = 0; i < NUM_SS_SNAPSHOT_XMIN_CACHE_PARTITIONS; i++) { + LWLock* partition_lock = SSSnapshotXminHashPartitionLockByIndex(i); + LWLockAcquire(partition_lock, LW_EXCLUSIVE); + } + while ((xmin_item = (ss_snap_xmin_item_t*)hash_seq_search(&hash_seq)) != NULL) { if (TimestampDifferenceExceeds(xmin_item->timestamp, cur_time, DMS_MSG_MAX_WAIT_TIME)) { ss_snap_xmin_key_t key{.xmin = xmin_item->xmin}; - hash_search(snap_cache, &key, HASH_REMOVE, NULL); + if (hash_search(snap_cache, &key, HASH_REMOVE, NULL) == NULL) { + ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("snapshot xmin cache hash table corrupted"))); + } continue; } @@ -86,21 +95,24 @@ void MaintXminInPrimary(void) } } - if (snap_xmin == MaxTransactionId) { - TimestampTz recent_time = pg_atomic_read_u64((volatile uint64*)&xmin_info->recent_snap_send_time); - if (TimestampDifferenceExceeds(cur_time, recent_time, 0)) { - return; - } + for (int i = NUM_SS_SNAPSHOT_XMIN_CACHE_PARTITIONS - 1; i >= 0; i--) { + LWLock* partition_lock = SSSnapshotXminHashPartitionLockByIndex(i); + LWLockRelease(partition_lock); } - xmin_info->snap_oldest_xmin = snap_xmin; uint64 new_global_xmin = GetOldestXminInNodeTable(); if (TransactionIdPrecedes(snap_xmin, new_global_xmin)) { new_global_xmin = snap_xmin; } + if (new_global_xmin == MaxTransactionId) { + return; + } + SpinLockAcquire(&xmin_info->global_oldest_xmin_lock); - xmin_info->global_oldest_xmin = new_global_xmin; + if (xmin_info->global_oldest_xmin_active) { + xmin_info->global_oldest_xmin = new_global_xmin; + } SpinLockRelease(&xmin_info->global_oldest_xmin_lock); } @@ -127,35 +139,36 @@ bool RecordSnapshotBeforeSend(uint8 inst_id, uint64 xmin) } ss_snap_xmin_key_t key = {.xmin = xmin}; + uint32 key_hash = SSSnapshotXminKeyHashCode(&key); + LWLock *partition_lock = SSSnapshotXminHashPartitionLock(key_hash); + LWLockAcquire(partition_lock, LW_EXCLUSIVE); ss_snap_xmin_item_t *xmin_item = (ss_snap_xmin_item_t*)hash_search(snap_cache, &key, HASH_ENTER_NULL, NULL); if (xmin_item == NULL) { + LWLockRelease(partition_lock); ereport(WARNING, (errmodule(MOD_DMS), errmsg("insert snapshot into snap_cache table failed, " "capacity is not enough"))); return false; } xmin_item->xmin = xmin; TimestampTz send_time = GetCurrentTimestamp(); - bool ret = false; - do { - uint64 recent_time = pg_atomic_read_u64((volatile uint64*)&xmin_info->recent_snap_send_time); - if ((uint64)send_time <= recent_time) { - break; - } - ret = pg_atomic_compare_exchange_u64((volatile uint64*)&xmin_info->recent_snap_send_time, &recent_time, send_time); - } while (!ret); xmin_item->timestamp = send_time; + LWLockRelease(partition_lock); return true; } uint64 SSGetGlobalOldestXmin(uint64 globalxmin) { ss_xmin_info_t *xmin_info = &g_instance.dms_cxt.SSXminInfo; + uint64 ret_globalxmin = globalxmin; + SpinLockAcquire(&xmin_info->global_oldest_xmin_lock); if (!xmin_info->global_oldest_xmin_active) { - return globalxmin; + if (TransactionIdPrecedes(xmin_info->prev_global_oldest_xmin, globalxmin)) { + ret_globalxmin = xmin_info->prev_global_oldest_xmin; + } + SpinLockRelease(&xmin_info->global_oldest_xmin_lock); + return ret_globalxmin; } - uint64 ret_globalxmin = globalxmin; - SpinLockAcquire(&xmin_info->global_oldest_xmin_lock); if (TransactionIdPrecedes(xmin_info->global_oldest_xmin, globalxmin)) { ret_globalxmin = xmin_info->global_oldest_xmin; } @@ -194,7 +207,9 @@ void SSUpdateNodeOldestXmin(uint8 inst_id, unsigned long long oldest_xmin) SpinLockAcquire(&xmin_info->bitmap_active_nodes_lock); if (xmin_info->bitmap_active_nodes == reform_info->bitmap_nodes) { + SpinLockAcquire(&xmin_info->global_oldest_xmin_lock); xmin_info->global_oldest_xmin_active = true; + SpinLockRelease(&xmin_info->global_oldest_xmin_lock); } SpinLockRelease(&xmin_info->bitmap_active_nodes_lock); } diff --git a/src/gausskernel/process/threadpool/knl_instance.cpp b/src/gausskernel/process/threadpool/knl_instance.cpp index 92ef81cb2..756c32229 100755 --- a/src/gausskernel/process/threadpool/knl_instance.cpp +++ b/src/gausskernel/process/threadpool/knl_instance.cpp @@ -212,9 +212,9 @@ static void knl_g_dms_init(knl_g_dms_context *dms_cxt) } xmin_info->snap_cache = NULL; xmin_info->snap_oldest_xmin = MaxTransactionId; - xmin_info->recent_snap_send_time = 0; SpinLockInit(&xmin_info->global_oldest_xmin_lock); xmin_info->global_oldest_xmin = MaxTransactionId; + xmin_info->prev_global_oldest_xmin = MaxTransactionId; xmin_info->global_oldest_xmin_active = false; SpinLockInit(&xmin_info->bitmap_active_nodes_lock); xmin_info->bitmap_active_nodes = 0; diff --git a/src/gausskernel/storage/lmgr/lwlock.cpp b/src/gausskernel/storage/lmgr/lwlock.cpp index 7d852b5cd..f0691fa8f 100644 --- a/src/gausskernel/storage/lmgr/lwlock.cpp +++ b/src/gausskernel/storage/lmgr/lwlock.cpp @@ -198,7 +198,8 @@ static const char *BuiltinTrancheNames[] = { "AuditIndextblLock", "PCABufferContentLock", "XlogTrackPartLock", - "SSTxnStatusCachePartLock" + "SSTxnStatusCachePartLock", + "SSSnapshotXminCachePartLock" }; static void RegisterLWLockTranches(void); @@ -662,6 +663,10 @@ static void InitializeLWLocks(int numLocks) LWLockInitialize(&lock->lock, LWTRANCHE_SS_TXNSTATUS_PARTITION); } + for (id = 0; id < NUM_SS_SNAPSHOT_XMIN_CACHE_PARTITIONS; id++, lock++) { + LWLockInitialize(&lock->lock, LWTRANCHE_SS_SNAPSHOT_XMIN_PARTITION); + } + Assert((lock - t_thrd.shemem_ptr_cxt.mainLWLockArray) == NumFixedLWLocks); for (id = NumFixedLWLocks; id < numLocks; id++, lock++) { diff --git a/src/include/ddes/dms/ss_xmin.h b/src/include/ddes/dms/ss_xmin.h index 9e15b9e64..bad467482 100644 --- a/src/include/ddes/dms/ss_xmin.h +++ b/src/include/ddes/dms/ss_xmin.h @@ -50,14 +50,20 @@ typedef struct st_ss_xmin_info { ss_node_xmin_item_t node_table[DMS_MAX_INSTANCES]; struct HTAB* snap_cache; uint64 snap_oldest_xmin; - volatile TimestampTz recent_snap_send_time; slock_t global_oldest_xmin_lock; uint64 global_oldest_xmin; + uint64 prev_global_oldest_xmin; bool global_oldest_xmin_active; slock_t bitmap_active_nodes_lock; uint64 bitmap_active_nodes; } ss_xmin_info_t; +#define SSSnapshotXminHashPartition(hashcode) ((hashcode) % NUM_SS_SNAPSHOT_XMIN_CACHE_PARTITIONS) +#define SSSnapshotXminHashPartitionLock(hashcode) \ + (&t_thrd.shemem_ptr_cxt.mainLWLockArray[FirstSSSnapshotXminCacheLock + SSSnapshotXminHashPartition(hashcode)].lock) +#define SSSnapshotXminHashPartitionLockByIndex(i) \ + (&t_thrd.shemem_ptr_cxt.mainLWLockArray[FirstSSSnapshotXminCacheLock + (i)].lock) +uint32 SSSnapshotXminKeyHashCode(const ss_snap_xmin_key_t *key); void MaintXminInPrimary(void); void MaintXminInStandby(void); bool RecordSnapshotBeforeSend(uint8 inst_id, uint64 xmin); diff --git a/src/include/storage/lock/lwlock.h b/src/include/storage/lock/lwlock.h index 94db36d51..dbd394e64 100644 --- a/src/include/storage/lock/lwlock.h +++ b/src/include/storage/lock/lwlock.h @@ -151,6 +151,9 @@ const struct LWLOCK_PARTITION_DESC LWLockPartInfo[] = { /* Number of partitions of the txnstatus mapping hashtable */ #define NUM_TXNSTATUS_CACHE_PARTITIONS 256 +/* Number of partitions of the snapshot xmin cache hashtable */ +#define NUM_SS_SNAPSHOT_XMIN_CACHE_PARTITIONS 32 + /* * WARNING---Please keep the order of LWLockTrunkOffset and BuiltinTrancheIds consistent!!! */ @@ -200,8 +203,10 @@ const struct LWLOCK_PARTITION_DESC LWLockPartInfo[] = { /* txn status cache */ #define FirstTxnStatusCacheLock (FirstXlogTrackLock + NUM_XLOG_TRACK_PARTITIONS) +/* shared-storage snapshot xmin cache*/ +#define FirstSSSnapshotXminCacheLock (FirstTxnStatusCacheLock + NUM_TXNSTATUS_CACHE_PARTITIONS) /* must be last: */ -#define NumFixedLWLocks (FirstTxnStatusCacheLock + NUM_TXNSTATUS_CACHE_PARTITIONS) +#define NumFixedLWLocks (FirstSSSnapshotXminCacheLock + NUM_SS_SNAPSHOT_XMIN_CACHE_PARTITIONS) /* * WARNING----Please keep BuiltinTrancheIds and BuiltinTrancheNames consistent!!! * @@ -282,6 +287,7 @@ enum BuiltinTrancheIds LWTRANCHE_PCA_BUFFER_CONTENT, LWTRANCHE_XLOG_TRACK_PARTITION, LWTRANCHE_SS_TXNSTATUS_PARTITION, + LWTRANCHE_SS_SNAPSHOT_XMIN_PARTITION, /* * Each trancheId above should have a corresponding item in BuiltinTrancheNames; */ From 9aaeea4032a74bdaef9a105d9b23bf0c09daf49e Mon Sep 17 00:00:00 2001 From: chendong76 <1209756284@qq.com> Date: Wed, 6 Sep 2023 17:29:46 +0800 Subject: [PATCH 203/304] =?UTF-8?q?=E3=80=90bugfix=E3=80=91=E8=A7=A3?= =?UTF-8?q?=E5=86=B3=E6=8C=89=E9=9C=80=E5=9B=9E=E6=94=BExlog=E6=97=A5?= =?UTF-8?q?=E5=BF=97=E9=87=8D=E5=A4=8D=E5=9B=9E=E6=94=BE=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ondemand_extreme_rto/page_redo.cpp | 118 ++++++++++++++++++ .../ondemand_extreme_rto/redo_utils.cpp | 25 +--- .../access/ondemand_extreme_rto/page_redo.h | 3 + 3 files changed, 125 insertions(+), 21 deletions(-) diff --git a/src/gausskernel/storage/access/transam/ondemand_extreme_rto/page_redo.cpp b/src/gausskernel/storage/access/transam/ondemand_extreme_rto/page_redo.cpp index 4456afbd5..56ea53b33 100644 --- a/src/gausskernel/storage/access/transam/ondemand_extreme_rto/page_redo.cpp +++ b/src/gausskernel/storage/access/transam/ondemand_extreme_rto/page_redo.cpp @@ -1582,6 +1582,100 @@ void RedoPageWorkerRedoBcmBlock(XLogRecParseState *procState) } } +// if holdLock is false, only lock if need redo; otherwise, lock anyway +bool checkBlockRedoDoneFromHashMapAndLock(LWLock **lock, RedoItemTag redoItemTag, RedoItemHashEntry **redoItemEntry, + bool holdLock) +{ + bool hashFound = false; + uint32 id = GetSlotId(redoItemTag.rNode, 0, 0, GetBatchCount()); + HTAB *hashMap = g_instance.comm_cxt.predo_cxt.redoItemHash[id]; + if (hashMap == NULL) { + ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("redo item hash table corrupted, there has invalid hashtable."))); + } + + unsigned int new_hash = XlogTrackTableHashCode(&redoItemTag); + *lock = XlogTrackMappingPartitionLock(new_hash); + (void)LWLockAcquire(*lock, LW_SHARED); + RedoItemHashEntry *entry = (RedoItemHashEntry *)hash_search(hashMap, (void *)&redoItemTag, HASH_FIND, &hashFound); + + /* Page is already up-to-date, no need to replay. */ + if (!hashFound || entry->redoItemNum == 0 || entry->redoDone) { + if (!holdLock) { + LWLockRelease(*lock); + *lock = NULL; + } + return true; + } + + // switch to exclusive lock in replay + LWLockRelease(*lock); + (void)LWLockAcquire(*lock, LW_EXCLUSIVE); + + // check again + if (entry->redoItemNum == 0 || entry->redoDone) { + if (!holdLock) { + LWLockRelease(*lock); + *lock = NULL; + } + return true; + } + + if (redoItemEntry != NULL) { + *redoItemEntry = entry; + } + return false; +} + +static inline void XLogRecGetRedoItemTag(XLogRecParseState *redoblockstate, RedoItemTag *redoItemTag) +{ + XLogBlockParse *blockparse = &(redoblockstate->blockparse); + + redoItemTag->rNode.dbNode = blockparse->blockhead.dbNode; + redoItemTag->rNode.relNode = blockparse->blockhead.relNode; + redoItemTag->rNode.spcNode = blockparse->blockhead.spcNode; + redoItemTag->rNode.bucketNode = blockparse->blockhead.bucketNode; + redoItemTag->rNode.opt = blockparse->blockhead.opt; + + redoItemTag->forkNum = blockparse->blockhead.forknum; + redoItemTag->blockNum = blockparse->blockhead.blkno; +} + +static inline bool IsXLogRecSameRedoBlock(XLogRecParseState *redoblockstate1, XLogRecParseState *redoblockstate2) +{ + RedoItemTag redoItemTag1; + RedoItemTag redoItemTag2; + + if (redoblockstate1 == NULL || redoblockstate2 == NULL) { + return false; + } + + XLogRecGetRedoItemTag(redoblockstate1, &redoItemTag1); + XLogRecGetRedoItemTag(redoblockstate2, &redoItemTag2); + + if (memcmp(&redoItemTag1, &redoItemTag2, sizeof(RedoItemTag)) != 0) { + return false; + } + return true; +} + +static inline bool IsProcInHashMap(XLogRecParseState *procState) +{ + bool result = false; + switch (XLogBlockHeadGetValidInfo(&procState->blockparse.blockhead)) { + case BLOCK_DATA_MAIN_DATA_TYPE: + case BLOCK_DATA_UNDO_TYPE: + case BLOCK_DATA_VM_TYPE: + case BLOCK_DATA_FSM_TYPE: + result = true; + break; + default: + break; + } + + return result; +} + void RedoPageWorkerMain() { (void)RegisterRedoInterruptCallBack(HandlePageRedoInterrupts); @@ -1592,6 +1686,7 @@ void RedoPageWorkerMain() } XLogRecParseState *redoblockstateHead = NULL; + LWLock *xlog_partition_lock = NULL; GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_1]); while ((redoblockstateHead = (XLogRecParseState *)SPSCBlockingQueueTop(g_redoWorker->queue)) != (XLogRecParseState *)&g_redoEndMark) { @@ -1623,6 +1718,7 @@ void RedoPageWorkerMain() bool notfound = false; bool updateFsm = false; bool needRelease = true; + bool redoDone = false; XLogRecParseState *procState = redoblockstateHead; XLogRecParseState *reloadBlockState = NULL; @@ -1634,6 +1730,21 @@ void RedoPageWorkerMain() // nextrecord will be redo in backwards position procState = (procState->distributeStatus == XLOG_TAIL_DISTRIBUTE) ? NULL : (XLogRecParseState *)procState->nextrecord; + if (xlog_partition_lock == NULL && SS_ONDEMAND_BUILD_DONE && IsProcInHashMap(redoblockstate)) { + RedoItemTag redoItemTag; + XLogRecGetRedoItemTag(redoblockstate, &redoItemTag); + redoDone = checkBlockRedoDoneFromHashMapAndLock(&xlog_partition_lock, redoItemTag, NULL, true); + } + + if (redoDone) { + Assert(xlog_partition_lock != NULL); + needRelease = false; + DereferenceRecParseState(redoblockstate); + SetCompletedReadEndPtr(g_redoWorker, redoblockstate->blockparse.blockhead.start_ptr, + redoblockstate->blockparse.blockhead.end_ptr); + goto redo_done; + } + switch (XLogBlockHeadGetValidInfo(&redoblockstate->blockparse.blockhead)) { case BLOCK_DATA_MAIN_DATA_TYPE: case BLOCK_DATA_UNDO_TYPE: @@ -1699,6 +1810,13 @@ void RedoPageWorkerMain() default: break; } + +redo_done: + if (xlog_partition_lock != NULL && !IsXLogRecSameRedoBlock(redoblockstate, procState)) { + LWLockRelease(xlog_partition_lock); + xlog_partition_lock = NULL; + redoDone = false; + } } (void)MemoryContextSwitchTo(oldCtx); GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_7]); diff --git a/src/gausskernel/storage/access/transam/ondemand_extreme_rto/redo_utils.cpp b/src/gausskernel/storage/access/transam/ondemand_extreme_rto/redo_utils.cpp index c93e275f0..c4b9bb5cd 100644 --- a/src/gausskernel/storage/access/transam/ondemand_extreme_rto/redo_utils.cpp +++ b/src/gausskernel/storage/access/transam/ondemand_extreme_rto/redo_utils.cpp @@ -23,6 +23,7 @@ #include "access/xlogproc.h" #include "access/ondemand_extreme_rto/batch_redo.h" +#include "access/ondemand_extreme_rto/page_redo.h" #include "access/ondemand_extreme_rto/dispatcher.h" #include "access/ondemand_extreme_rto/redo_utils.h" #include "access/ondemand_extreme_rto/xlog_read.h" @@ -243,10 +244,8 @@ void OndemandXLogParseBufferRelease(XLogRecParseState *recordstate) BufferDesc *RedoForOndemandExtremeRTOQuery(BufferDesc *bufHdr, char relpersistence, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode) { - bool hashFound = false; bool needMarkDirty = false; - unsigned int new_hash; - LWLock *xlog_partition_lock; + LWLock *xlog_partition_lock = NULL; Buffer buf = BufferDescriptorGetBuffer(bufHdr); ondemand_extreme_rto::RedoItemHashEntry *redoItemEntry = NULL; ondemand_extreme_rto::RedoItemTag redoItemTag; @@ -259,27 +258,11 @@ BufferDesc *RedoForOndemandExtremeRTOQuery(BufferDesc *bufHdr, char relpersisten INIT_REDO_ITEM_TAG(redoItemTag, bufHdr->tag.rnode, forkNum, blockNum); - uint32 id = ondemand_extreme_rto::GetSlotId(bufHdr->tag.rnode, 0, 0, ondemand_extreme_rto::GetBatchCount()); - HTAB *hashMap = g_instance.comm_cxt.predo_cxt.redoItemHash[id]; - if (hashMap == NULL) { - ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), - errmsg("redo item hash table corrupted, there has invalid hashtable."))); - } - - new_hash = ondemand_extreme_rto::XlogTrackTableHashCode(&redoItemTag); - xlog_partition_lock = XlogTrackMappingPartitionLock(new_hash); - (void)LWLockAcquire(xlog_partition_lock, LW_SHARED); - redoItemEntry = (ondemand_extreme_rto::RedoItemHashEntry *)hash_search(hashMap, (void *)&redoItemTag, HASH_FIND, &hashFound); - - /* Page is already up-to-date, no need to replay. */ - if (!hashFound || redoItemEntry->redoItemNum == 0 || redoItemEntry->redoDone) { - LWLockRelease(xlog_partition_lock); + if (checkBlockRedoDoneFromHashMapAndLock(&xlog_partition_lock, redoItemTag, &redoItemEntry, false)) { return bufHdr; } - // switch to exclusive lock in replay - LWLockRelease(xlog_partition_lock); - (void)LWLockAcquire(xlog_partition_lock, LW_EXCLUSIVE); + Assert(xlog_partition_lock != NULL); rc = memset_s(&bufferInfo, sizeof(bufferInfo), 0, sizeof(bufferInfo)); securec_check(rc, "\0", "\0"); diff --git a/src/include/access/ondemand_extreme_rto/page_redo.h b/src/include/access/ondemand_extreme_rto/page_redo.h index 285d66c2a..1615a3683 100644 --- a/src/include/access/ondemand_extreme_rto/page_redo.h +++ b/src/include/access/ondemand_extreme_rto/page_redo.h @@ -30,6 +30,7 @@ #include "knl/knl_variable.h" #include "access/ondemand_extreme_rto/redo_item.h" +#include "access/ondemand_extreme_rto/batch_redo.h" #include "nodes/pg_list.h" #include "storage/proc.h" @@ -247,6 +248,8 @@ void BatchClearRecoveryThreadHashTbl(Oid spcNode, Oid dbNode); void RecordBadBlockAndPushToRemote(XLogBlockDataParse *datadecode, PageErrorType error_type, XLogRecPtr old_lsn, XLogPhyBlock pblk); const char *RedoWokerRole2Str(RedoRole role); +bool checkBlockRedoDoneFromHashMapAndLock(LWLock **lock, RedoItemTag redoItemTag, RedoItemHashEntry **redoItemEntry, + bool holdLock); } // namespace ondemand_extreme_rto #endif From acd9bf5f3c58d78f8140592be3a85f5af9da4e43 Mon Sep 17 00:00:00 2001 From: hemny Date: Wed, 6 Sep 2023 20:13:40 +0800 Subject: [PATCH 204/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8DSIGUSR2=E4=BF=A1?= =?UTF-8?q?=E5=8F=B7=E5=B1=8F=E8=94=BD=E4=B8=A2=E5=A4=B1=E5=BC=95=E8=B5=B7?= =?UTF-8?q?=E7=9A=84=E6=AD=BB=E9=94=81=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/process/postmaster/aiocompleter.cpp | 2 +- src/gausskernel/process/postmaster/postmaster.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/gausskernel/process/postmaster/aiocompleter.cpp b/src/gausskernel/process/postmaster/aiocompleter.cpp index aa0d16ca2..c58da3da2 100644 --- a/src/gausskernel/process/postmaster/aiocompleter.cpp +++ b/src/gausskernel/process/postmaster/aiocompleter.cpp @@ -612,7 +612,7 @@ static void CompltrConfig(SIGNAL_ARGS) */ static void CompltrQuickDie(SIGNAL_ARGS) { - PG_SETMASK(&t_thrd.libpq_cxt.BlockSig); + gs_signal_setmask(&t_thrd.libpq_cxt.BlockSig, NULL); /* * We DO NOT want to run proc_exit() callbacks -- we're here because diff --git a/src/gausskernel/process/postmaster/postmaster.cpp b/src/gausskernel/process/postmaster/postmaster.cpp index 7cb582977..618b342ee 100644 --- a/src/gausskernel/process/postmaster/postmaster.cpp +++ b/src/gausskernel/process/postmaster/postmaster.cpp @@ -5662,7 +5662,7 @@ static void SIGHUP_handler(SIGNAL_ARGS) */ if (get_file_lock(gucconf_lock_file, &filelock) != CODE_OK) { ereport(WARNING, (errmsg("the last sigup signal is processing,get file lock failed."))); - (void)PG_SETMASK(&t_thrd.libpq_cxt.UnBlockSig); + gs_signal_setmask(&t_thrd.libpq_cxt.UnBlockSig, NULL); errno = save_errno; return; } @@ -5677,7 +5677,7 @@ static void SIGHUP_handler(SIGNAL_ARGS) } if (!get_locked) { ereport(WARNING, (errmsg("the last sigup signal is processing,get file thread lock failed."))); - (void)PG_SETMASK(&t_thrd.libpq_cxt.UnBlockSig); + gs_signal_setmask(&t_thrd.libpq_cxt.UnBlockSig, NULL); errno = save_errno; return; } From 6ab62032c4ab7fbf3726900760e9b3f49ffb8ce0 Mon Sep 17 00:00:00 2001 From: yaojun Date: Tue, 5 Sep 2023 16:22:53 +0800 Subject: [PATCH 205/304] feat: add dss io statistics function Signed-off-by: yaojun --- src/common/backend/catalog/builtin_funcs.ini | 4 + src/common/backend/utils/adt/pgstatfuncs.cpp | 45 +++++++++ src/common/backend/utils/init/globals.cpp | 2 +- src/gausskernel/storage/file/fio_device.cpp | 2 +- .../rollback-post_catalog_maindb_92_910.sql | 1 + .../rollback-post_catalog_otherdb_92_910.sql | 1 + .../upgrade-post_catalog_maindb_92_910.sql | 10 ++ .../upgrade-post_catalog_otherdb_92_910.sql | 10 ++ src/include/storage/file/fio_device.h | 94 +++++++++++++++++-- src/include/utils/builtins.h | 2 +- 10 files changed, 160 insertions(+), 11 deletions(-) create mode 100644 src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback-post_catalog_maindb_92_910.sql create mode 100644 src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback-post_catalog_otherdb_92_910.sql create mode 100644 src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_910.sql create mode 100644 src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_910.sql diff --git a/src/common/backend/catalog/builtin_funcs.ini b/src/common/backend/catalog/builtin_funcs.ini index 77c2dc72f..2e1100759 100755 --- a/src/common/backend/catalog/builtin_funcs.ini +++ b/src/common/backend/catalog/builtin_funcs.ini @@ -2500,6 +2500,10 @@ "dsqrt", 1, AddBuiltinFunc(_0(230), _1("dsqrt"), _2(1), _3(true), _4(false), _5(dsqrt), _6(701), _7(PG_CATALOG_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(0), _12(0), _13(0), _14(false), _15(false), _16(false), _17(false), _18('i'), _19(0), _20(1, 701), _21(NULL), _22(NULL), _23(NULL), _24(NULL), _25("dsqrt"), _26(NULL), _27(NULL), _28(NULL), _29(0), _30(false), _31(NULL), _32(false), _33("square root"), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)) ), + AddFuncGroup( + "dss_io_stat", 1, + AddBuiltinFunc(_0(6990), _1("dss_io_stat"), _2(1), _3(true), _4(false), _5(dss_io_stat), _6(2249), _7(PG_CATALOG_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(0), _12(0), _13(0), _14(false), _15(false), _16(false), _17(false), _18('i'), _19(0), _20(1, INT4OID), _21(4, INT4OID, INT8OID, INT8OID, INT4OID), _22(4, 'i', 'o', 'o', 'o'), _23(4, "duration", "read_kilobyte_per_sec", "write_kilobyte_per_sec", "io_times"), _24(NULL), _25("dss_io_stat"), _26(NULL), _27(NULL), _28(NULL), _29(0), _30(false), _31(NULL), _32(false), _33(NULL), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)) + ), AddFuncGroup( "dsynonym_init", 1, AddBuiltinFunc(_0(3728), _1("dsynonym_init"), _2(1), _3(true), _4(false), _5(dsynonym_init), _6(2281), _7(PG_CATALOG_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(0), _12(0), _13(0), _14(false), _15(false), _16(false), _17(false), _18('i'), _19(0), _20(1, 2281), _21(NULL), _22(NULL), _23(NULL), _24(NULL), _25("dsynonym_init"), _26(NULL), _27(NULL), _28(NULL), _29(0), _30(false), _31(NULL), _32(false), _33("(internal)"), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)) diff --git a/src/common/backend/utils/adt/pgstatfuncs.cpp b/src/common/backend/utils/adt/pgstatfuncs.cpp index df34a6015..eedbb0a57 100644 --- a/src/common/backend/utils/adt/pgstatfuncs.cpp +++ b/src/common/backend/utils/adt/pgstatfuncs.cpp @@ -78,6 +78,7 @@ #include "storage/lock/lock.h" #include "nodes/makefuncs.h" #include "ddes/dms/ss_dms_bufmgr.h" +#include "storage/file/fio_device.h" #define UINT32_ACCESS_ONCE(var) ((uint32)(*((volatile uint32*)&(var)))) #define NUM_PG_LOCKTAG_ID 12 @@ -87,6 +88,8 @@ #define UPPERCASE_LETTERS_ID 55 #define LOWERCASE_LETTERS_ID 87 #define DISPLACEMENTS_VALUE 32 +#define MAX_DURATION_TIME 60 +#define DSS_IO_STAT_COLUMN_NUM 3 const uint32 INDEX_STATUS_VIEW_COL_NUM = 3; @@ -14723,6 +14726,48 @@ Datum track_memory_context_detail(PG_FUNCTION_ARGS) } } + +/* + * @Description : Get the statistical information for DSS IO, including read bytes, write bytes and io times. + * @in : Duration of statistics. + * @out : None. + * @return : Node. + */ +Datum dss_io_stat(PG_FUNCTION_ARGS) +{ + if (!ENABLE_DMS) { + ereport(ERROR, (errmsg("This function only supports shared storage."))); + } + Datum result; + TupleDesc tupdesc; + int32 duration = PG_GETARG_INT32(0); + if (duration > MAX_DURATION_TIME) { + ereport(ERROR, (errmsg("The duration is too long, and it must be less than 60s."))); + } + init_dss_io_stat(); + unsigned long long read_bytes = 0; + unsigned long long write_bytes = 0; + int io_count = 0; + get_dss_io_stat(duration, &read_bytes, &write_bytes, &io_count); + // tuple header + int i = 1; + tupdesc = CreateTemplateTupleDesc(DSS_IO_STAT_COLUMN_NUM, false); + TupleDescInitEntry(tupdesc, (AttrNumber)i++, "read_kilobyte_per_sec", INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)i++, "write_kilobyte_per_sec", INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)i, "io_times", INT4OID, -1, 0); + tupdesc = BlessTupleDesc(tupdesc); + // tuple body + Datum values[DSS_IO_STAT_COLUMN_NUM]; + bool nulls[DSS_IO_STAT_COLUMN_NUM]={false}; + i = 0; + values[i++] = UInt64GetDatum(read_bytes); + values[i++] = UInt64GetDatum(write_bytes); + values[i] = Int32GetDatum(io_count); + + HeapTuple heap_tuple = heap_form_tuple(tupdesc, values, nulls); + result = HeapTupleGetDatum(heap_tuple); + PG_RETURN_DATUM(result); +} #ifdef ENABLE_MULTIPLE_NODES /* Get the head row of the view of index status */ TupleDesc get_index_status_view_frist_row() diff --git a/src/common/backend/utils/init/globals.cpp b/src/common/backend/utils/init/globals.cpp index 1dd115185..51a9db748 100644 --- a/src/common/backend/utils/init/globals.cpp +++ b/src/common/backend/utils/init/globals.cpp @@ -75,7 +75,7 @@ bool will_shutdown = false; * NEXT | 92899 | ? | ? * ********************************************/ -const uint32 GRAND_VERSION_NUM = 92909; +const uint32 GRAND_VERSION_NUM = 92910; /******************************************** * 2.VERSION NUM FOR EACH FEATURE diff --git a/src/gausskernel/storage/file/fio_device.cpp b/src/gausskernel/storage/file/fio_device.cpp index 021351115..5ab740b95 100644 --- a/src/gausskernel/storage/file/fio_device.cpp +++ b/src/gausskernel/storage/file/fio_device.cpp @@ -28,7 +28,7 @@ #include "c.h" #include "storage/file/fio_device.h" - +g_dss_io_stat g_dss_io_stat_var; device_type_t fio_device_type(const char *name) { if (is_dss_file(name)) { diff --git a/src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback-post_catalog_maindb_92_910.sql b/src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback-post_catalog_maindb_92_910.sql new file mode 100644 index 000000000..2e9244f9a --- /dev/null +++ b/src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback-post_catalog_maindb_92_910.sql @@ -0,0 +1 @@ +DROP FUNCTION IF EXISTS pg_catalog.dss_io_stat() CASCADE; \ No newline at end of file diff --git a/src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback-post_catalog_otherdb_92_910.sql b/src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback-post_catalog_otherdb_92_910.sql new file mode 100644 index 000000000..2e9244f9a --- /dev/null +++ b/src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback-post_catalog_otherdb_92_910.sql @@ -0,0 +1 @@ +DROP FUNCTION IF EXISTS pg_catalog.dss_io_stat() CASCADE; \ No newline at end of file diff --git a/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_910.sql b/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_910.sql new file mode 100644 index 000000000..1966880b7 --- /dev/null +++ b/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_910.sql @@ -0,0 +1,10 @@ +DROP FUNCTION IF EXISTS pg_catalog.dss_io_stat() CASCADE; +SET LOCAL inplace_upgrade_next_system_object_oids=IUO_PROC, 6990; +CREATE FUNCTION pg_catalog.dss_io_stat +( + int4, + OUT read_kilobyte_per_sec int8, + OUT write_kilobyte_per_sec int8, + OUT io_times int4 +) +RETURNS record LANGUAGE INTERNAL as 'dss_io_stat'; \ No newline at end of file diff --git a/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_910.sql b/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_910.sql new file mode 100644 index 000000000..1966880b7 --- /dev/null +++ b/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_910.sql @@ -0,0 +1,10 @@ +DROP FUNCTION IF EXISTS pg_catalog.dss_io_stat() CASCADE; +SET LOCAL inplace_upgrade_next_system_object_oids=IUO_PROC, 6990; +CREATE FUNCTION pg_catalog.dss_io_stat +( + int4, + OUT read_kilobyte_per_sec int8, + OUT write_kilobyte_per_sec int8, + OUT io_times int4 +) +RETURNS record LANGUAGE INTERNAL as 'dss_io_stat'; \ No newline at end of file diff --git a/src/include/storage/file/fio_device.h b/src/include/storage/file/fio_device.h index b19048803..b8ae4613c 100644 --- a/src/include/storage/file/fio_device.h +++ b/src/include/storage/file/fio_device.h @@ -32,6 +32,7 @@ #include #include #include +#include #include "storage/dss/fio_dss.h" #include "storage/file/fio_device_com.h" @@ -94,40 +95,117 @@ static inline int close_dev(int fd) } } +typedef struct g_dss_io_stat { + int read_bytes; + unsigned long long write_bytes; + unsigned long long read_write_count; + bool is_ready_for_stat; + pthread_mutex_t lock; + g_dss_io_stat() { + pthread_mutex_init(&lock, NULL); + is_ready_for_stat = false; + read_bytes = 0; + write_bytes = 0; + read_write_count = 0; + } + ~g_dss_io_stat() { + pthread_mutex_destroy(&lock); + } +} g_dss_io_stat; + +extern g_dss_io_stat g_dss_io_stat_var; +/*Initialize global variable*/ +static inline void init_dss_io_stat() +{ + pthread_mutex_lock(&g_dss_io_stat_var.lock); + g_dss_io_stat_var.is_ready_for_stat = true; + g_dss_io_stat_var.read_bytes = 0; + g_dss_io_stat_var.write_bytes = 0; + g_dss_io_stat_var.read_write_count = 0; + +} + +#define KB 1024 +/* +* duration: statistical duration +* kB_read: total read kilobyte during the time +* kB_write: total write kilobyte during the time +* io_count: total read and write count +*/ +static inline void get_dss_io_stat(int duration, unsigned long long *kB_read, unsigned long long *kB_write, int *io_count) +{ + sleep(duration); + if (kB_read) { + *kB_read = g_dss_io_stat_var.read_bytes / duration / KB; + } + if (kB_write) { + *kB_write = g_dss_io_stat_var.write_bytes / duration / KB; + } + if (io_count) { + *io_count = g_dss_io_stat_var.read_write_count; + } + g_dss_io_stat_var.is_ready_for_stat = false; + pthread_mutex_unlock(&g_dss_io_stat_var.lock); +} + static inline ssize_t read_dev(int fd, void *buf, size_t count) { + ssize_t ret = 0; if (is_dss_fd(fd)) { - return dss_read_file(fd, buf, count); + ret = dss_read_file(fd, buf, count); } else { - return read(fd, buf, count); + ret = read(fd, buf, count); } + if (g_dss_io_stat_var.is_ready_for_stat) { + g_dss_io_stat_var.read_bytes += count; + g_dss_io_stat_var.read_write_count += 1; + } + return ret; } static inline ssize_t pread_dev(int fd, void *buf, size_t count, off_t offset) { + ssize_t ret = 0; if (is_dss_fd(fd)) { - return dss_pread_file(fd, buf, count, offset); + ret = dss_pread_file(fd, buf, count, offset); } else { - return pread(fd, buf, count, offset); + ret = pread(fd, buf, count, offset); + } + if (g_dss_io_stat_var.is_ready_for_stat) { + g_dss_io_stat_var.read_bytes += count; + g_dss_io_stat_var.read_write_count += 1; } + return ret; } static inline ssize_t write_dev(int fd, const void *buf, size_t count) { + ssize_t ret = 0; if (is_dss_fd(fd)) { - return dss_write_file(fd, buf, count); + ret = dss_write_file(fd, buf, count); } else { - return write(fd, buf, count); + ret = write(fd, buf, count); } + if (g_dss_io_stat_var.is_ready_for_stat) { + g_dss_io_stat_var.write_bytes += count; + g_dss_io_stat_var.read_write_count += 1; + } + return ret; } static inline ssize_t pwrite_dev(int fd, const void *buf, size_t count, off_t offset) { + ssize_t ret = 0; if (is_dss_fd(fd)) { - return dss_pwrite_file(fd, buf, count, offset); + ret = dss_pwrite_file(fd, buf, count, offset); } else { - return pwrite(fd, buf, count, offset); + ret = pwrite(fd, buf, count, offset); + } + if (g_dss_io_stat_var.is_ready_for_stat) { + g_dss_io_stat_var.write_bytes += count; + g_dss_io_stat_var.read_write_count += 1; } + return ret; } static inline off_t lseek_dev(int fd, off_t offset, int whence) diff --git a/src/include/utils/builtins.h b/src/include/utils/builtins.h index ad380a880..54629770e 100644 --- a/src/include/utils/builtins.h +++ b/src/include/utils/builtins.h @@ -1858,7 +1858,7 @@ extern Datum compress_buffer_stat_info(PG_FUNCTION_ARGS); extern Datum compress_ratio_info(PG_FUNCTION_ARGS); extern Datum compress_statistic_info(PG_FUNCTION_ARGS); extern Datum pg_read_binary_file_blocks(PG_FUNCTION_ARGS); - +extern Datum dss_io_stat(PG_FUNCTION_ARGS); #else #endif extern char *pg_ultostr(char *str, uint32 value); From 6945e809765b7f174e4f47a661c93e207f84e08b Mon Sep 17 00:00:00 2001 From: jiangzhaoheng Date: Mon, 14 Aug 2023 17:34:04 +0800 Subject: [PATCH 206/304] =?UTF-8?q?=E5=85=B3=E9=97=ADustore=E5=BC=95?= =?UTF-8?q?=E6=93=8E=E5=90=8E=EF=BC=8C=E5=88=86=E9=85=8Dxid=E4=B8=8D?= =?UTF-8?q?=E5=86=8D=E5=88=86=E9=85=8Dundozone?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/storage/access/ustore/undo/knl_uundoapi.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/gausskernel/storage/access/ustore/undo/knl_uundoapi.cpp b/src/gausskernel/storage/access/ustore/undo/knl_uundoapi.cpp index bb72f6e27..b36a67921 100644 --- a/src/gausskernel/storage/access/ustore/undo/knl_uundoapi.cpp +++ b/src/gausskernel/storage/access/ustore/undo/knl_uundoapi.cpp @@ -647,6 +647,9 @@ void RecoveryUndoSystemMeta(void) void AllocateUndoZone() { #ifndef ENABLE_MULTIPLE_NODES + if (!g_instance.attr.attr_storage.enable_ustore) { + return; + } AllocateZonesBeforXid(); #endif } From 17cc3cca3f730f924ffad3e4a9f316255f8731aa Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Thu, 7 Sep 2023 15:24:58 +0800 Subject: [PATCH 207/304] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E5=8F=8C=E9=9B=86?= =?UTF-8?q?=E7=BE=A4=E6=97=A5=E5=BF=97=E5=9B=9E=E6=94=B6=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/storage/access/transam/xlog.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/gausskernel/storage/access/transam/xlog.cpp b/src/gausskernel/storage/access/transam/xlog.cpp index 28fe66427..eea9c2ee7 100755 --- a/src/gausskernel/storage/access/transam/xlog.cpp +++ b/src/gausskernel/storage/access/transam/xlog.cpp @@ -8942,6 +8942,8 @@ void StartupXLOG(void) ereport(LOG, (errmsg("[On-demand]: Ondemand recovery do not finish in last reform, " "reading control file of original primary:%d", src_id))); SSOndemandRecoveryExitNormal = false; + } else if (SS_REPLICATION_DORADO_CLUSTER) { + src_id = SSGetPrimaryInstId(); } else { if (SS_STANDBY_FAILOVER || SS_STANDBY_PROMOTING) { src_id = SSGetPrimaryInstId(); @@ -13116,7 +13118,7 @@ static void KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo, XLogRecPtr curIns * while dms and dss enable, t_thrd.xlog_cxt.server_mode only is normal_mode, we do additional * check for dms and dss enabling. */ - if (!ENABLE_DMS && t_thrd.xlog_cxt.server_mode == PRIMARY_MODE) { + if ((!ENABLE_DMS && t_thrd.xlog_cxt.server_mode == PRIMARY_MODE) || SS_REPLICATION_PRIMARY_NODE) { if (WalSndInProgress(SNDROLE_PRIMARY_BUILDSTANDBY) || pg_atomic_read_u32(&g_instance.comm_cxt.current_gsrewind_count) > 0) { /* segno = 1 show all file should be keep */ From 8752c51169d38eef6ab161befdea02e603028d3d Mon Sep 17 00:00:00 2001 From: nnuanyang Date: Wed, 30 Aug 2023 20:01:49 -0700 Subject: [PATCH 208/304] mysql_error bug --- src/common/backend/utils/error/elog.cpp | 45 ++++++- src/common/pl/plpgsql/src/pl_exec.cpp | 38 +++--- src/include/utils/elog.h | 2 + src/include/utils/plpgsql.h | 2 +- src/test/regress/expected/mysql_condition.out | 48 +++++-- src/test/regress/expected/mysql_resignal.out | 120 +++++++++--------- src/test/regress/expected/mysql_signal.out | 120 +++++++++--------- src/test/regress/sql/mysql_condition.sql | 13 ++ 8 files changed, 234 insertions(+), 154 deletions(-) diff --git a/src/common/backend/utils/error/elog.cpp b/src/common/backend/utils/error/elog.cpp index 24fa62bf1..a255eb3a5 100644 --- a/src/common/backend/utils/error/elog.cpp +++ b/src/common/backend/utils/error/elog.cpp @@ -494,6 +494,7 @@ bool errstart(int elevel, const char* filename, int lineno, const char* funcname edata->table_name = NULL; edata->column_name = NULL; edata->cursor_name = NULL; + edata->mysql_errno = NULL; t_thrd.log_cxt.recursion_depth--; return true; @@ -729,6 +730,8 @@ void errfinish(int dummy, ...) pfree(edata->column_name); if (edata->cursor_name) pfree(edata->cursor_name); + if (edata->mysql_errno) + pfree(edata->mysql_errno); t_thrd.log_cxt.errordata_stack_depth--; /* Exit error-handling context */ @@ -1665,6 +1668,27 @@ int signal_cursor_name(const char *cursor_name) return 0; /* return value does not matter */ } +/* + * signal_mysql_errno --- add mysql_errno to the current error + */ +int signal_mysql_errno(const char *mysql_errno) +{ + ErrorData* edata = &t_thrd.log_cxt.errordata[t_thrd.log_cxt.errordata_stack_depth]; + + /* we don't bother incrementing t_thrd.log_cxt.recursion_depth */ + CHECK_STACK_DEPTH(); + + if (edata->mysql_errno != NULL) { + pfree(edata->mysql_errno); + edata->mysql_errno = NULL; + } + + if (mysql_errno != NULL) { + edata->mysql_errno = MemoryContextStrdup(ErrorContext, mysql_errno); + } + + return 0; /* return value does not matter */ +} /* * ErrOutToClient --- sets whether to send error output to client or not. */ @@ -2030,6 +2054,8 @@ ErrorData* CopyErrorData(void) newedata->column_name = pstrdup(newedata->column_name); if (newedata->cursor_name) newedata->cursor_name = pstrdup(newedata->cursor_name); + if (newedata->mysql_errno) + newedata->mysql_errno = pstrdup(newedata->mysql_errno); return newedata; } @@ -2058,6 +2084,7 @@ void UpdateErrorData(ErrorData* edata, ErrorData* newData) FREE_POINTER(edata->table_name); FREE_POINTER(edata->column_name); FREE_POINTER(edata->cursor_name); + FREE_POINTER(edata->mysql_errno); MemoryContext oldcontext = MemoryContextSwitchTo(ErrorContext); edata->elevel = newData->elevel; @@ -2089,6 +2116,7 @@ void UpdateErrorData(ErrorData* edata, ErrorData* newData) edata->table_name = pstrdup(newData->table_name); edata->column_name = pstrdup(newData->column_name); edata->cursor_name = pstrdup(newData->cursor_name); + edata->mysql_errno = pstrdup(newData->mysql_errno); MemoryContextSwitchTo(oldcontext); } @@ -2172,6 +2200,10 @@ void FreeErrorData(ErrorData* edata) pfree(edata->cursor_name); edata->cursor_name = NULL; } + if (edata->mysql_errno) { + pfree(edata->mysql_errno); + edata->mysql_errno = NULL; + } pfree(edata); } @@ -2283,6 +2315,8 @@ void ReThrowError(ErrorData* edata) newedata->column_name = pstrdup(newedata->column_name); if (newedata->cursor_name) newedata->cursor_name = pstrdup(newedata->cursor_name); + if (newedata->mysql_errno) + newedata->mysql_errno = pstrdup(newedata->mysql_errno); t_thrd.log_cxt.recursion_depth--; if (DB_IS_CMPT(B_FORMAT)) { @@ -5639,6 +5673,9 @@ static char* mask_Password_internal(const char* query_string) if (NULL != edata->cursor_name) { pfree_ext(edata->cursor_name); } + if (NULL != edata->mysql_errno) { + pfree_ext(edata->mysql_errno); + } t_thrd.log_cxt.errordata_stack_depth--; } } @@ -6012,10 +6049,10 @@ void pushErrorData(ErrorData *edata) dolphinErrorData->class_origin = pstrdup(edata->class_origin); dolphinErrorData->subclass_origin = pstrdup(edata->subclass_origin); dolphinErrorData->sqlstatestr = pstrdup(edata->sqlstate); - char sqlerrcode [128]; - int ret = sprintf_s(sqlerrcode, 128, "%d", edata->sqlerrcode); - securec_check_ss_c(ret, "\0", "\0"); - dolphinErrorData->errorcode = pstrdup(sqlerrcode); + if (edata->mysql_errno == NULL) + dolphinErrorData->errorcode = pstrdup(plpgsql_get_sqlstate(edata->sqlerrcode)); + else + dolphinErrorData->errorcode = pstrdup(edata->mysql_errno); } else { dolphinErrorData->class_origin = pstrdup(class_origin); dolphinErrorData->subclass_origin = pstrdup(subclass_origin); diff --git a/src/common/pl/plpgsql/src/pl_exec.cpp b/src/common/pl/plpgsql/src/pl_exec.cpp index 5e8bb96f3..680f65415 100644 --- a/src/common/pl/plpgsql/src/pl_exec.cpp +++ b/src/common/pl/plpgsql/src/pl_exec.cpp @@ -4118,7 +4118,8 @@ static int exec_stmt(PLpgSQL_execstate* estate, PLpgSQL_stmt* stmt, bool resigna if ((enum PLpgSQL_stmt_types)stmt->cmd_type == PLPGSQL_STMT_BLOCK) { if (estate->handler_level == estate->block_level) { copyDiffErrorDataArea(u_sess->dolphin_errdata_ctx.errorDataArea, u_sess->dolphin_errdata_ctx.lastErrorDataArea, estate->cur_error); - copyErrorDataArea(u_sess->dolphin_errdata_ctx.lastErrorDataArea, u_sess->dolphin_errdata_ctx.errorDataArea); + if (u_sess->dolphin_errdata_ctx.lastErrorDataArea->current_edata_count != 0) + copyErrorDataArea(u_sess->dolphin_errdata_ctx.lastErrorDataArea, u_sess->dolphin_errdata_ctx.errorDataArea); u_sess->dolphin_errdata_ctx.handler_active = false; } else if (!u_sess->dolphin_errdata_ctx.handler_active) { copyErrorDataArea(u_sess->dolphin_errdata_ctx.errorDataArea, u_sess->dolphin_errdata_ctx.lastErrorDataArea); @@ -4406,7 +4407,7 @@ static int exec_stmt_b_getdiag(PLpgSQL_execstate* estate, PLpgSQL_stmt_getdiag* edata->sqlerrcode = ERRCODE_INVALID_CONDITION_NUMBER; edata->message = "Invalid condition number"; edata->class_origin = edata->sqlstate = edata->subclass_origin = edata->cons_catalog = edata->cons_schema = NULL; - edata->cons_name = edata->catalog_name = edata->schema_name = edata->table_name = edata->column_name = edata->cursor_name = NULL; + edata->cons_name = edata->catalog_name = edata->schema_name = edata->table_name = edata->column_name = edata->cursor_name = edata->mysql_errno = NULL; copyErrorDataArea(u_sess->dolphin_errdata_ctx.lastErrorDataArea, u_sess->dolphin_errdata_ctx.errorDataArea); pushErrorData(edata); pfree_ext(edata); @@ -6380,6 +6381,7 @@ static void exec_get_condition_information(PLpgSQL_execstate* estate, PLpgSQL_st PLpgSQL_condition_info_item *con_item) { ListCell *lc = NULL; + int code = 0; foreach (lc, stmt->cond_info_item) { PLpgSQL_signal_info_item *item = (PLpgSQL_signal_info_item *)lfirst(lc); @@ -6408,12 +6410,13 @@ static void exec_get_condition_information(PLpgSQL_execstate* estate, PLpgSQL_st con_item->message_text = pstrdup(extval); break; case PLPGSQL_MYSQL_ERRNO: - con_item->sqlerrcode = pg_atoi(extval, sizeof(int32), false); - if (con_item->sqlerrcode <= 0 || con_item->sqlerrcode > MYSQL_ERRNO_MAX) { + code = pg_atoi(extval, sizeof(int32), false); + if (code <= 0 || code > MYSQL_ERRNO_MAX) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("Variable '%s' can't be set to the value of '%s'", item->con_name, extval))); } + con_item->sqlerrcode = pstrdup(extval); break; case PLPGSQL_CONSTRAINT_CATALOG: con_item->constraint_catalog = pstrdup(extval); @@ -6451,10 +6454,10 @@ static void exec_get_condition_information(PLpgSQL_execstate* estate, PLpgSQL_st return; } -static void StoreSignalError(int elevel, PLpgSQL_condition_info_item *con_item, bool is_warning_throw, int is_signal) +static void StoreSignalError(int elevel, int code, PLpgSQL_condition_info_item *con_item, bool is_warning_throw, int is_signal) { ereport(elevel, - (errcode(con_item->sqlerrcode ? con_item->sqlerrcode : 0), + (errcode(code), errmsg_internal("%s", con_item->message_text), (con_item->sqlstate != NULL) ? signal_returnd_sqlstate(con_item->sqlstate) : 0, (con_item->class_origin != NULL) ? signal_class_origin(con_item->class_origin) : 0, @@ -6467,6 +6470,7 @@ static void StoreSignalError(int elevel, PLpgSQL_condition_info_item *con_item, (con_item->table_name != NULL) ? signal_table_name(con_item->table_name) : 0, (con_item->column_name != NULL) ? signal_column_name(con_item->column_name) : 0, (con_item->cursor_name != NULL) ? signal_cursor_name(con_item->cursor_name) : 0, + (con_item->sqlerrcode != NULL) ? signal_mysql_errno(con_item->sqlerrcode) : 0, signal_is_warnings_throw(is_warning_throw), signal_is_signal(is_signal))); @@ -6486,6 +6490,7 @@ static void exec_free_con_item(PLpgSQL_condition_info_item *con_item) FREE_POINTER(con_item->table_name); FREE_POINTER(con_item->column_name); FREE_POINTER(con_item->cursor_name); + FREE_POINTER(con_item->sqlerrcode); FREE_POINTER(con_item); } @@ -6502,7 +6507,6 @@ static int exec_stmt_signal(PLpgSQL_execstate* estate, PLpgSQL_stmt_signal* stmt bool is_warning_throw = false;; PLpgSQL_condition_info_item *con_item = (PLpgSQL_condition_info_item *)palloc0(sizeof(PLpgSQL_condition_info_item)); - con_item->sqlerrcode = stmt->sqlerrstate; /* sqlsate is not null */ if (sqlstate == NULL) { @@ -6515,20 +6519,20 @@ static int exec_stmt_signal(PLpgSQL_execstate* estate, PLpgSQL_stmt_signal* stmt if (sqlstate[0] == '0' && sqlstate[1] == '1') { con_item->message_text = pstrdup("Unhandled user-defined warning condition"); - con_item->sqlerrcode = MAKE_SQLSTATE('0', '1', '0', '0', '0'); + con_item->sqlerrcode = pstrdup("01000"); elevel = WARNING; is_warning_throw = is_declare_handler; } else if (sqlstate[0] == '0' && sqlstate[1] == '2') { con_item->message_text = pstrdup("Unhandled user-defined not found condition"); - con_item->sqlerrcode = MAKE_SQLSTATE('0', '2', '0', '0', '0'); + con_item->sqlerrcode = pstrdup("02000"); } else { con_item->message_text = pstrdup("Unhandled user-defined exception condition"); - con_item->sqlerrcode = MAKE_SQLSTATE('0', '3', '0', '0', '0'); + con_item->sqlerrcode = pstrdup("03000"); } exec_get_condition_information(estate, stmt, con_item); - StoreSignalError(elevel, con_item, is_warning_throw, PLpgSQL_signal_resignal::PLPGSQL_SIGNAL); + StoreSignalError(elevel, stmt->sqlerrstate, con_item, is_warning_throw, PLpgSQL_signal_resignal::PLPGSQL_SIGNAL); exec_free_con_item(con_item); return PLPGSQL_RC_OK; @@ -6550,7 +6554,7 @@ static int exec_stmt_resignal(PLpgSQL_execstate* estate, PLpgSQL_stmt_signal* st PLpgSQL_condition_info_item *con_item = (PLpgSQL_condition_info_item *)palloc0(sizeof(PLpgSQL_condition_info_item)); con_item->message_text = pstrdup(cur_errdata->message); - con_item->sqlerrcode = cur_errdata->sqlerrcode; + con_item->sqlerrcode = pstrdup(cur_errdata->mysql_errno); if (sqlstate != NULL) { has_sqlstate = true; con_item->sqlstate = pstrdup(sqlstate); @@ -6560,22 +6564,22 @@ static int exec_stmt_resignal(PLpgSQL_execstate* estate, PLpgSQL_stmt_signal* st if (sqlstate != NULL) { if (sqlstate[0] == '0' && sqlstate[1] == '1') { - con_item->sqlerrcode = MAKE_SQLSTATE('0', '1', '0', '0', '0'); + con_item->sqlerrcode = pstrdup("01000"); elevel = WARNING; is_warning_throw = is_declare_handler; } else if (sqlstate[0] == '0' && sqlstate[1] == '2') { - con_item->sqlerrcode = MAKE_SQLSTATE('0', '2', '0', '0', '0'); + con_item->sqlerrcode = pstrdup("02000"); } else { - con_item->sqlerrcode = MAKE_SQLSTATE('0', '3', '0', '0', '0'); + con_item->sqlerrcode = pstrdup("03000"); } } exec_get_condition_information(estate, stmt, con_item); if (has_sqlstate) { - StoreSignalError(elevel, con_item, is_warning_throw, PLpgSQL_signal_resignal::PLPGSQL_RESIGNAL_WITH_SQLSTATE); + StoreSignalError(elevel, cur_errdata->sqlerrcode, con_item, is_warning_throw, PLpgSQL_signal_resignal::PLPGSQL_RESIGNAL_WITH_SQLSTATE); } else { - StoreSignalError(elevel, con_item, is_warning_throw, PLpgSQL_signal_resignal::PLPGSQL_RESIGNAL_WITHOUT_SQLSTATE); + StoreSignalError(elevel, cur_errdata->sqlerrcode, con_item, is_warning_throw, PLpgSQL_signal_resignal::PLPGSQL_RESIGNAL_WITHOUT_SQLSTATE); } exec_free_con_item(con_item); diff --git a/src/include/utils/elog.h b/src/include/utils/elog.h index bc24df861..78a474dc0 100644 --- a/src/include/utils/elog.h +++ b/src/include/utils/elog.h @@ -260,6 +260,7 @@ extern int signal_schema_name(const char *schema_name); extern int signal_table_name(const char *table_name); extern int signal_column_name(const char *column_name); extern int signal_cursor_name(const char *cursor_name); +extern int signal_mysql_errno(const char *mysql_errno); extern int signal_is_signal(int is_signal); extern void save_error_message(void); @@ -535,6 +536,7 @@ typedef struct ErrorData { char* table_name; /* table_name for signal/resignal */ char* column_name; /* column_name for signal/resignal */ char* cursor_name; /* cursor_name for signal/resignal */ + char* mysql_errno; /* mysql_errno for signal/resignal */ bool is_warnings_throw; int is_signal; } ErrorData; diff --git a/src/include/utils/plpgsql.h b/src/include/utils/plpgsql.h index 22f4e7a99..b579514f4 100644 --- a/src/include/utils/plpgsql.h +++ b/src/include/utils/plpgsql.h @@ -972,7 +972,7 @@ typedef struct { /* condition information item name for signal/resignal */ char *table_name; char *column_name; char *cursor_name; - int sqlerrcode; /* mysql_errno */ + char *sqlerrcode; /* mysql_errno */ } PLpgSQL_condition_info_item; typedef struct { /* siganl_information_item */ diff --git a/src/test/regress/expected/mysql_condition.out b/src/test/regress/expected/mysql_condition.out index 596c2f440..e8fa6682b 100644 --- a/src/test/regress/expected/mysql_condition.out +++ b/src/test/regress/expected/mysql_condition.out @@ -973,9 +973,10 @@ call prc(); (1 row) show warnings; - level | code | message --------+------+--------- -(0 rows) + level | code | message +-------+-------+-------------------------------------------------------- + Error | 23502 | null value in column "c1" violates not-null constraint +(1 row) select @retSqlstate, @msg; @retsqlstate | @msg @@ -1234,13 +1235,13 @@ select @num1,@num2,@num3,@num4; select @msg1,@errno1; @msg1 | @errno1 ------------------------------------------+--------- - Unhandled user-defined warning condition | 64 + Unhandled user-defined warning condition | 01000 (1 row) select @msg2,@errno2; @msg2 | @errno2 ------------------------------------------+--------- - Unhandled user-defined warning condition | 64 + Unhandled user-defined warning condition | 01000 (1 row) select @msg3,@errno3; @@ -1302,13 +1303,13 @@ select @num1,@num2,@num3,@num4; select @msg1,@errno1; @msg1 | @errno1 ------------------------------------------+--------- - Unhandled user-defined warning condition | 64 + Unhandled user-defined warning condition | 01000 (1 row) select @msg2,@errno2; @msg2 | @errno2 ------------------------------------------+--------- - Unhandled user-defined warning condition | 64 + Unhandled user-defined warning condition | 01000 (1 row) select @msg3,@errno3; @@ -1320,7 +1321,7 @@ select @msg3,@errno3; select @msg4,@errno4; @msg4 | @errno4 ------------------------------------------+--------- - Unhandled user-defined warning condition | 64 + Unhandled user-defined warning condition | 01000 (1 row) CREATE or replace PROCEDURE p1() IS @@ -1370,13 +1371,13 @@ select @num1,@num2,@num3,@num4; select @msg1,@errno1; @msg1 | @errno1 ------------------------------------------+--------- - Unhandled user-defined warning condition | 64 + Unhandled user-defined warning condition | 01000 (1 row) select @msg2,@errno2; @msg2 | @errno2 ------------------------------------------+--------- - Unhandled user-defined warning condition | 64 + Unhandled user-defined warning condition | 01000 (1 row) select @msg3,@errno3; @@ -1388,7 +1389,7 @@ select @msg3,@errno3; select @msg4,@errno4; @msg4 | @errno4 ------------------------------------------+--------- - Unhandled user-defined warning condition | 64 + Unhandled user-defined warning condition | 01000 (1 row) drop procedure p1; @@ -1473,7 +1474,7 @@ SELECT @returned_sqlstate; @class_origin | @subclass_origin | @constraint_catalog | @constraint_schema | @constraint_name | @catalog_name | @schema_name | @table_name | @column_name | @cursor_name | @message_text | @mysql_errno | @returned_sqlstate ---------------+------------------+---------------------+--------------------+------------------+---------------+--------------+-------------+--------------+--------------+-----------------+--------------+-------------------- - (null) | (null) | (null) | (null) | (null) | (null) | (null) | (null) | (null) | (null) | inout parameter | 64 | 01234 + (null) | (null) | (null) | (null) | (null) | (null) | (null) | (null) | (null) | (null) | inout parameter | 01000 | 01234 (1 row) set @class_origin='',@subclass_origin='',@returned_sqlstate='',@message_text= '',@mysql_errno='',@constraint_catalog='',@constraint_schema='',@constraint_name='',@catalog_name='',@schema_name='',@table_name='',@column_name='',@cursor_name=''; @@ -1669,6 +1670,29 @@ CONTEXT: referenced column: stacked_diagnostics_test (1 row) +CREATE OR REPLACE PROCEDURE p_resig1() IS +begin +DECLARE EXIT HANDLER FOR SQLSTATE '42P01' +BEGIN +RESIGNAL; +END; +DROP TABLE t1; +end; +/ +call p_resig1(); + p_resig1 +---------- + +(1 row) + +get diagnostics condition 1 @p1 = CLASS_ORIGIN,@p2 = SUBCLASS_ORIGIN,@p3 = MESSAGE_TEXT,@p4 = MYSQL_ERRNO,@p5 = CONSTRAINT_CATALOG,@p6 = CONSTRAINT_SCHEMA, +@p7 = CONSTRAINT_NAME,@p8 = CATALOG_NAME,@p9 = SCHEMA_NAME,@p10 = TABLE_NAME,@p11 = COLUMN_NAME,@p12 = CURSOR_NAME; +select @p1,@p2,@p3,@p4; + @p1 | @p2 | @p3 | @p4 +-----+-----+-----+----- + | | | +(1 row) + \c regression -- test access to exception data create function zero_divide() returns int as $$ diff --git a/src/test/regress/expected/mysql_resignal.out b/src/test/regress/expected/mysql_resignal.out index c4a7f82f3..255b8f767 100644 --- a/src/test/regress/expected/mysql_resignal.out +++ b/src/test/regress/expected/mysql_resignal.out @@ -355,9 +355,9 @@ call p1(); ERROR: table "t1" does not exist CONTEXT: PL/pgSQL function p1() line 4 at RESIGNAL show warnings; - level | code | message --------+----------+--------------------------- - Error | 16908420 | table "t1" does not exist + level | code | message +-------+-------+--------------------------- + Error | 42P01 | table "t1" does not exist (1 row) DROP TABLE IF EXISTS t1; @@ -375,9 +375,9 @@ call p1(); ERROR: table "t1" does not exist CONTEXT: PL/pgSQL function p1() line 4 at RESIGNAL show warnings; - level | code | message --------+----------+--------------------------- - Error | 16908420 | table "t1" does not exist + level | code | message +-------+-------+--------------------------- + Error | 42P01 | table "t1" does not exist (1 row) CREATE OR REPLACE PROCEDURE p1() IS @@ -398,9 +398,9 @@ CONTEXT: PL/pgSQL function p1() line 4 at RESIGNAL (1 row) show warnings; - level | code | message ----------+------+------------------------------------------ - Warning | 64 | Unhandled user-defined warning condition + level | code | message +---------+-------+------------------------------------------ + Warning | 01000 | Unhandled user-defined warning condition (1 row) CREATE OR REPLACE PROCEDURE p1() IS @@ -416,9 +416,9 @@ call p1(); ERROR: Unhandled user-defined not found condition CONTEXT: PL/pgSQL function p1() line 4 at RESIGNAL show warnings; - level | code | message --------+------+-------------------------------------------- - Error | 128 | Unhandled user-defined not found condition + level | code | message +-------+-------+-------------------------------------------- + Error | 02000 | Unhandled user-defined not found condition (1 row) DROP TABLE IF EXISTS t1; @@ -437,9 +437,9 @@ call p1(); ERROR: table "t1" does not exist CONTEXT: PL/pgSQL function p1() line 5 at RESIGNAL show warnings; - level | code | message --------+----------+--------------------------- - Error | 16908420 | table "t1" does not exist + level | code | message +-------+-------+--------------------------- + Error | 42P01 | table "t1" does not exist (1 row) DROP TABLE IF EXISTS t1; @@ -642,7 +642,7 @@ show warnings; level | code | message -------+-------+--------------------------- Error | 42P01 | table "t1" does not exist - Error | 192 | table is not defined + Error | 03000 | table is not defined (2 rows) DROP TABLE IF EXISTS t1; @@ -663,7 +663,7 @@ show warnings; level | code | message -------+-------+--------------------------- Error | 42P01 | table "t1" does not exist - Error | 192 | table "t1" does not exist + Error | 03000 | table "t1" does not exist (2 rows) DROP TABLE IF EXISTS t1; @@ -684,7 +684,7 @@ show warnings; level | code | message -------+-------+--------------------------- Error | 42P01 | table "t1" does not exist - Error | 192 | table "t1" does not exist + Error | 03000 | table "t1" does not exist (2 rows) DROP TABLE IF EXISTS t1; @@ -1045,10 +1045,10 @@ CONTEXT: PL/pgSQL function p1() line 3 at RESIGNAL (1 row) show warnings; - level | code | message ----------+------+------------------------------------------ - Warning | 64 | Unhandled user-defined warning condition - Warning | 1 | this is warnings + level | code | message +---------+-------+------------------------------------------ + Warning | 01000 | Unhandled user-defined warning condition + Warning | 1 | this is warnings (2 rows) CREATE OR REPLACE PROCEDURE p1(a out text) IS @@ -1335,8 +1335,8 @@ show warnings; level | code | message ---------+-------+--------------------------- Error | 42P01 | table "t1" does not exist - Warning | 64 | column is not defined2222 - Warning | 64 | column is not defined + Warning | 01000 | column is not defined2222 + Warning | 01000 | column is not defined (3 rows) DROP TABLE IF EXISTS t1; @@ -1366,8 +1366,8 @@ show warnings; level | code | message ---------+-------+--------------------------- Error | 42P01 | table "t1" does not exist - Warning | 64 | column is not defined2222 - Warning | 64 | column is not defined + Warning | 01000 | column is not defined2222 + Warning | 01000 | column is not defined (3 rows) DROP TABLE IF EXISTS t1; @@ -1397,8 +1397,8 @@ show warnings; level | code | message ---------+-------+--------------------------- Error | 42P01 | table "t1" does not exist - Error | 192 | column is not defined2222 - Warning | 64 | column is not defined + Error | 03000 | column is not defined2222 + Warning | 01000 | column is not defined (3 rows) DROP TABLE IF EXISTS t1; @@ -1516,8 +1516,8 @@ show warnings; level | code | message ---------+-------+--------------------------- Error | 42P01 | table "t1" does not exist - Warning | 64 | column is not defined2222 - Warning | 64 | column is not defined + Warning | 01000 | column is not defined2222 + Warning | 01000 | column is not defined (3 rows) DROP TABLE IF EXISTS t1; @@ -1607,9 +1607,9 @@ call p2(); ERROR: table "t1" does not exist CONTEXT: PL/pgSQL function p2() line 6 at RESIGNAL show warnings; - level | code | message --------+----------+--------------------------- - Error | 16908420 | table "t1" does not exist + level | code | message +-------+-------+--------------------------- + Error | 42P01 | table "t1" does not exist (1 row) select @a, @error_count; @@ -1638,9 +1638,9 @@ call p2(); ERROR: table "t1" does not exist CONTEXT: PL/pgSQL function p2() line 6 at RESIGNAL show warnings; - level | code | message --------+----------+--------------------------- - Error | 16908420 | table "t1" does not exist + level | code | message +-------+-------+--------------------------- + Error | 42P01 | table "t1" does not exist (1 row) select @a, @error_count; @@ -1732,9 +1732,9 @@ call p2(); ERROR: table "t1" does not exist CONTEXT: PL/pgSQL function p2() line 6 at RESIGNAL show warnings; - level | code | message --------+----------+--------------------------- - Error | 16908420 | table "t1" does not exist + level | code | message +-------+-------+--------------------------- + Error | 42P01 | table "t1" does not exist (1 row) select @a, @error_count; @@ -1763,9 +1763,9 @@ call p2(); ERROR: table "t1" does not exist CONTEXT: PL/pgSQL function p2() line 6 at RESIGNAL show warnings; - level | code | message --------+----------+--------------------------- - Error | 16908420 | table "t1" does not exist + level | code | message +-------+-------+--------------------------- + Error | 42P01 | table "t1" does not exist (1 row) select @a, @error_count; @@ -1794,9 +1794,9 @@ call p2(); ERROR: table "t1" does not exist CONTEXT: PL/pgSQL function p2() line 6 at RESIGNAL show warnings; - level | code | message --------+----------+--------------------------- - Error | 16908420 | table "t1" does not exist + level | code | message +-------+-------+--------------------------- + Error | 42P01 | table "t1" does not exist (1 row) select @a, @error_count; @@ -1825,9 +1825,9 @@ call p2(); ERROR: table "t1" does not exist CONTEXT: PL/pgSQL function p2() line 6 at RESIGNAL show warnings; - level | code | message --------+----------+--------------------------- - Error | 16908420 | table "t1" does not exist + level | code | message +-------+-------+--------------------------- + Error | 42P01 | table "t1" does not exist (1 row) select @a, @error_count; @@ -1856,9 +1856,9 @@ call p2(); ERROR: table "t1" does not exist CONTEXT: PL/pgSQL function p2() line 6 at RESIGNAL show warnings; - level | code | message --------+----------+--------------------------- - Error | 16908420 | table "t1" does not exist + level | code | message +-------+-------+--------------------------- + Error | 42P01 | table "t1" does not exist (1 row) select @a, @error_count; @@ -1887,9 +1887,9 @@ call p2(); ERROR: table "t1" does not exist CONTEXT: PL/pgSQL function p2() line 6 at RESIGNAL show warnings; - level | code | message --------+----------+--------------------------- - Error | 16908420 | table "t1" does not exist + level | code | message +-------+-------+--------------------------- + Error | 42P01 | table "t1" does not exist (1 row) select @a, @error_count; @@ -1918,9 +1918,9 @@ call p2(); ERROR: table "t1" does not exist CONTEXT: PL/pgSQL function p2() line 6 at RESIGNAL show warnings; - level | code | message --------+----------+--------------------------- - Error | 16908420 | table "t1" does not exist + level | code | message +-------+-------+--------------------------- + Error | 42P01 | table "t1" does not exist (1 row) select @a, @error_count; @@ -1949,9 +1949,9 @@ call p2(); ERROR: table "t1" does not exist CONTEXT: PL/pgSQL function p2() line 6 at RESIGNAL show warnings; - level | code | message --------+----------+--------------------------- - Error | 16908420 | table "t1" does not exist + level | code | message +-------+-------+--------------------------- + Error | 42P01 | table "t1" does not exist (1 row) select @a, @error_count; diff --git a/src/test/regress/expected/mysql_signal.out b/src/test/regress/expected/mysql_signal.out index ec5a527fa..239846df6 100644 --- a/src/test/regress/expected/mysql_signal.out +++ b/src/test/regress/expected/mysql_signal.out @@ -137,9 +137,9 @@ CONTEXT: PL/pgSQL function p1() line 2 at SIGNAL (1 row) show warnings; - level | code | message ----------+------+------------------------------------------ - Warning | 64 | Unhandled user-defined warning condition + level | code | message +---------+-------+------------------------------------------ + Warning | 01000 | Unhandled user-defined warning condition (1 row) CREATE OR REPLACE PROCEDURE p1() IS @@ -151,9 +151,9 @@ call p1(); ERROR: Unhandled user-defined not found condition CONTEXT: PL/pgSQL function p1() line 2 at SIGNAL show warnings; - level | code | message --------+------+-------------------------------------------- - Error | 128 | Unhandled user-defined not found condition + level | code | message +-------+-------+-------------------------------------------- + Error | 02000 | Unhandled user-defined not found condition (1 row) CREATE OR REPLACE PROCEDURE p1() IS @@ -165,9 +165,9 @@ call p1(); ERROR: Unhandled user-defined exception condition CONTEXT: PL/pgSQL function p1() line 2 at SIGNAL show warnings; - level | code | message --------+------+-------------------------------------------- - Error | 192 | Unhandled user-defined exception condition + level | code | message +-------+-------+-------------------------------------------- + Error | 03000 | Unhandled user-defined exception condition (1 row) CREATE OR REPLACE PROCEDURE p1() IS @@ -179,9 +179,9 @@ call p1(); ERROR: Unhandled user-defined exception condition CONTEXT: PL/pgSQL function p1() line 2 at SIGNAL show warnings; - level | code | message --------+------+-------------------------------------------- - Error | 192 | Unhandled user-defined exception condition + level | code | message +-------+-------+-------------------------------------------- + Error | 03000 | Unhandled user-defined exception condition (1 row) -- parse error @@ -219,9 +219,9 @@ call p1(); ERROR: Unhandled user-defined exception condition CONTEXT: PL/pgSQL function p1() line 3 at SIGNAL show warnings; - level | code | message --------+------+-------------------------------------------- - Error | 192 | Unhandled user-defined exception condition + level | code | message +-------+-------+-------------------------------------------- + Error | 03000 | Unhandled user-defined exception condition (1 row) CREATE OR REPLACE PROCEDURE p1() IS @@ -239,9 +239,9 @@ CONTEXT: PL/pgSQL function p1() line 3 at SIGNAL (1 row) show warnings; - level | code | message ----------+------+------------------------------------------ - Warning | 64 | Unhandled user-defined warning condition + level | code | message +---------+-------+------------------------------------------ + Warning | 01000 | Unhandled user-defined warning condition (1 row) CREATE OR REPLACE PROCEDURE p1() IS @@ -254,9 +254,9 @@ call p1(); ERROR: Unhandled user-defined not found condition CONTEXT: PL/pgSQL function p1() line 3 at SIGNAL show warnings; - level | code | message --------+------+-------------------------------------------- - Error | 128 | Unhandled user-defined not found condition + level | code | message +-------+-------+-------------------------------------------- + Error | 02000 | Unhandled user-defined not found condition (1 row) -- parse_error @@ -280,9 +280,9 @@ CONTEXT: PL/pgSQL function p1() line 2 at SIGNAL ERROR: Unhandled user-defined not found condition CONTEXT: PL/pgSQL function p1() line 3 at SIGNAL show warnings; - level | code | message --------+------+-------------------------------------------- - Error | 128 | Unhandled user-defined not found condition + level | code | message +-------+-------+-------------------------------------------- + Error | 02000 | Unhandled user-defined not found condition (1 row) -- 2.signa with SQLSTATE and signal_information_item @@ -629,9 +629,9 @@ call p1(0); ERROR: Unhandled user-defined exception condition CONTEXT: PL/pgSQL function p1() line 2 at SIGNAL show warnings; - level | code | message --------+------+-------------------------------------------- - Error | 192 | Unhandled user-defined exception condition + level | code | message +-------+-------+-------------------------------------------- + Error | 03000 | Unhandled user-defined exception condition (1 row) DROP TABLE IF EXISTS t1; @@ -647,9 +647,9 @@ call p1(0); ERROR: Unhandled user-defined exception condition CONTEXT: PL/pgSQL function p1() line 2 at SIGNAL show warnings; - level | code | message --------+------+-------------------------------------------- - Error | 192 | Unhandled user-defined exception condition + level | code | message +-------+-------+-------------------------------------------- + Error | 03000 | Unhandled user-defined exception condition (1 row) DROP TABLE IF EXISTS t1; @@ -665,9 +665,9 @@ call p1(0); ERROR: the table is not exist CONTEXT: PL/pgSQL function p1() line 2 at SIGNAL show warnings; - level | code | message --------+------+------------------------ - Error | 192 | the table is not exist + level | code | message +-------+-------+------------------------ + Error | 03000 | the table is not exist (1 row) DROP TABLE IF EXISTS t1; @@ -709,9 +709,9 @@ call p1(0); ERROR: Unhandled user-defined exception condition CONTEXT: PL/pgSQL function p1() line 5 at SIGNAL show warnings; - level | code | message --------+------+-------------------------------------------- - Error | 192 | Unhandled user-defined exception condition + level | code | message +-------+-------+-------------------------------------------- + Error | 03000 | Unhandled user-defined exception condition (1 row) DROP TABLE IF EXISTS t1; @@ -1214,9 +1214,9 @@ CONTEXT: PL/pgSQL function p1() line 6 at SIGNAL (1 row) show warnings; - level | code | message ----------+------+----------------------- - Warning | 64 | column is not defined + level | code | message +---------+-------+----------------------- + Warning | 01000 | column is not defined (1 row) DROP TABLE IF EXISTS t1; @@ -1243,9 +1243,9 @@ CONTEXT: PL/pgSQL function p1() line 6 at SIGNAL (1 row) show warnings; - level | code | message ----------+------+----------------------- - Warning | 64 | column is not defined + level | code | message +---------+-------+----------------------- + Warning | 01000 | column is not defined (1 row) DROP TABLE IF EXISTS t1; @@ -1272,9 +1272,9 @@ CONTEXT: PL/pgSQL function p1() line 6 at SIGNAL (1 row) show warnings; - level | code | message ----------+------+----------------------- - Warning | 64 | column is not defined + level | code | message +---------+-------+----------------------- + Warning | 01000 | column is not defined (1 row) DROP TABLE IF EXISTS t1; @@ -1389,9 +1389,9 @@ CONTEXT: PL/pgSQL function p1() line 6 at SIGNAL (1 row) show warnings; - level | code | message ----------+------+----------------------- - Warning | 64 | column is not defined + level | code | message +---------+-------+----------------------- + Warning | 01000 | column is not defined (1 row) DROP TABLE IF EXISTS t1; @@ -1452,9 +1452,9 @@ call p2(); ERROR: an error occurred CONTEXT: PL/pgSQL function p2() line 4 at SIGNAL show warnings; - level | code | message --------+------+------------------- - Error | 192 | an error occurred + level | code | message +-------+-------+------------------- + Error | 03000 | an error occurred (1 row) CREATE OR REPLACE PROCEDURE p2(pval int) IS @@ -1481,27 +1481,27 @@ CONTEXT: PL/pgSQL function p2(integer) line 4 at SIGNAL (1 row) show warnings; - level | code | message ----------+------+------------------------------------------ - Warning | 64 | Unhandled user-defined warning condition + level | code | message +---------+-------+------------------------------------------ + Warning | 01000 | Unhandled user-defined warning condition (1 row) call p2(1); ERROR: an error occurred for 1 CONTEXT: PL/pgSQL function p2(integer) line 6 at SIGNAL show warnings; - level | code | message --------+------+------------------------- - Error | 192 | an error occurred for 1 + level | code | message +-------+-------+------------------------- + Error | 03000 | an error occurred for 1 (1 row) call p2(2); ERROR: an error occurred for 2 CONTEXT: PL/pgSQL function p2(integer) line 8 at SIGNAL show warnings; - level | code | message --------+------+------------------------- - Error | 192 | an error occurred for 2 + level | code | message +-------+-------+------------------------- + Error | 03000 | an error occurred for 2 (1 row) call p2(3); diff --git a/src/test/regress/sql/mysql_condition.sql b/src/test/regress/sql/mysql_condition.sql index f40ee0072..f208d80b0 100644 --- a/src/test/regress/sql/mysql_condition.sql +++ b/src/test/regress/sql/mysql_condition.sql @@ -1094,6 +1094,19 @@ end; $$ language plpgsql; select stacked_diagnostics_test(); +CREATE OR REPLACE PROCEDURE p_resig1() IS +begin +DECLARE EXIT HANDLER FOR SQLSTATE '42P01' +BEGIN +RESIGNAL; +END; +DROP TABLE t1; +end; +/ +call p_resig1(); +get diagnostics condition 1 @p1 = CLASS_ORIGIN,@p2 = SUBCLASS_ORIGIN,@p3 = MESSAGE_TEXT,@p4 = MYSQL_ERRNO,@p5 = CONSTRAINT_CATALOG,@p6 = CONSTRAINT_SCHEMA, +@p7 = CONSTRAINT_NAME,@p8 = CATALOG_NAME,@p9 = SCHEMA_NAME,@p10 = TABLE_NAME,@p11 = COLUMN_NAME,@p12 = CURSOR_NAME; +select @p1,@p2,@p3,@p4; \c regression -- test access to exception data create function zero_divide() returns int as $$ From ea1c30640e979a14431a968237dc6318d02833d1 Mon Sep 17 00:00:00 2001 From: z00848344 Date: Wed, 23 Aug 2023 15:00:37 +0800 Subject: [PATCH 209/304] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E6=AE=B5=E5=BC=8F?= =?UTF-8?q?=E5=AD=98=E5=82=A8=E4=B8=8A=E5=88=9B=E5=BB=BAhash=E7=B4=A2?= =?UTF-8?q?=E5=BC=95coredump=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../storage/access/hash/hashpage.cpp | 61 +++++++++++-------- src/test/regress/expected/hash_index_001.out | 50 +++++++++++++++ src/test/regress/sql/hash_index_001.sql | 25 ++++++++ 3 files changed, 109 insertions(+), 27 deletions(-) diff --git a/src/gausskernel/storage/access/hash/hashpage.cpp b/src/gausskernel/storage/access/hash/hashpage.cpp index c3b7afe93..aa96262c1 100644 --- a/src/gausskernel/storage/access/hash/hashpage.cpp +++ b/src/gausskernel/storage/access/hash/hashpage.cpp @@ -969,39 +969,46 @@ static bool _hash_alloc_buckets(Relation rel, BlockNumber firstblock, uint32 nbl if (lastblock < firstblock || lastblock == InvalidBlockNumber) return false; - if (IsSegmentFileNode(rel->rd_node)) { - Buffer buf = ReadBuffer(rel, P_NEW); -#ifdef USE_ASSERT_CHECKING - BufferDesc *buf_desc = GetBufferDescriptor(buf - 1); - Assert(buf_desc->tag.blockNum == lastblock); -#endif - ReleaseBuffer(buf); - } else { - page = (Page)zerobuf; - - /* - * Initialize the page. Just zeroing the page won't work; see - * _hash_freeovflpage for similar usage. We take care to make the special - * space valid for the benefit of tools such as pageinspect. - */ - _hash_pageinit(page, BLCKSZ); - - ovflopaque = (HashPageOpaque) PageGetSpecialPointer(page); - - ovflopaque->hasho_prevblkno = InvalidBlockNumber; - ovflopaque->hasho_nextblkno = InvalidBlockNumber; - ovflopaque->hasho_bucket = -1; - ovflopaque->hasho_flag = LH_UNUSED_PAGE; - ovflopaque->hasho_page_id = HASHO_PAGE_ID; - - if (RelationNeedsWAL(rel)) + /* change segment table insert hash table */ + page = (Page)zerobuf; + /* + * Initialize the page. Just zeroing the page won't work; see + * _hash_freeovflpage for similar usage. We take care to make the special + * space valid for the benefit of tools such as pageinspect. + */ + _hash_pageinit(page, BLCKSZ); + + ovflopaque = (HashPageOpaque) PageGetSpecialPointer(page); + + ovflopaque->hasho_prevblkno = InvalidBlockNumber; + ovflopaque->hasho_nextblkno = InvalidBlockNumber; + ovflopaque->hasho_bucket = -1; + ovflopaque->hasho_flag = LH_UNUSED_PAGE; + ovflopaque->hasho_page_id = HASHO_PAGE_ID; + PageSetChecksumInplace(zerobuf, lastblock); + + if (RelationNeedsWAL(rel)) log_newpage(&rel->rd_node, MAIN_FORKNUM, lastblock, zerobuf, true); + + if (IsSegmentFileNode(rel->rd_node)) { + Buffer buf; + for (int i = firstblock; i <= lastblock; i++) { + buf = ReadBuffer(rel, P_NEW); + ReleaseBuffer(buf); + } + buf = ReadBuffer(rel, lastblock); + LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); + errno_t rel = memcpy_s(BufferGetPage(buf), BLCKSZ, page, BLCKSZ); + securec_check(rel, "", ""); + MarkBufferDirty(buf); + LockBuffer(buf, BUFFER_LOCK_UNLOCK); + ReleaseBuffer(buf); + } else { RelationOpenSmgr(rel); - PageSetChecksumInplace(zerobuf, lastblock); smgrextend(rel->rd_smgr, MAIN_FORKNUM, lastblock, zerobuf, false); } diff --git a/src/test/regress/expected/hash_index_001.out b/src/test/regress/expected/hash_index_001.out index 31d036597..b0f7c85b5 100644 --- a/src/test/regress/expected/hash_index_001.out +++ b/src/test/regress/expected/hash_index_001.out @@ -240,3 +240,53 @@ select count(*) from hash_table_7; 1000 (1 row) +-- create hash index on segment table, update/delete +drop table if exists hash_table_8; +NOTICE: table "hash_table_8" does not exist, skipping +create table hash_table_8(id int, name varchar, sex varchar default 'male') with(segment = on); +create index hash_tb8_id1 on hash_table_8 using hash(id); +insert into hash_table_8 select generate_series(1, 1000), 'xxx', 'xxx'; +select count(*) from hash_table_8; + count +------- + 1000 +(1 row) + +update hash_table_8 set sex = tmp.sex from (values (10, 'xxx', 'female'), (20, 'xxx', 'female'), (30, 'xxx', 'female')) as tmp (id, name, sex) where hash_table_8.id = tmp.id; +select * from hash_table_8 where sex = 'female'; + id | name | sex +----+------+-------- + 10 | xxx | female + 20 | xxx | female + 30 | xxx | female +(3 rows) + +delete from hash_table_8 where sex = 'female'; +select * from hash_table_8 where sex = 'female'; + id | name | sex +----+------+----- +(0 rows) + +drop index hash_tb8_id1; +drop table hash_table_8; +-- create hash index on segment table, delete/vacuum +drop table if exists hash_table_9; +NOTICE: table "hash_table_9" does not exist, skipping +create table hash_table_9(id int, name varchar, sec varchar default 'male') with (segment = on); +create index hash_tb9_id1 on hash_table_9 using hash(id); +insert into hash_table_9 select generate_series(1, 1000), 'XXX', 'XXX'; +insert into hash_table_9 select generate_series(1, 200), 'AAA', 'AAA'; +select count(*) from hash_table_9 where name = 'AAA'; + count +------- + 200 +(1 row) + +delete from hash_table_9 where name = 'AAA'; +select * from hash_table_9 where name = 'AAA'; + id | name | sec +----+------+----- +(0 rows) + +drop index hash_tb9_id1; +drop table hash_table_9; diff --git a/src/test/regress/sql/hash_index_001.sql b/src/test/regress/sql/hash_index_001.sql index dc7fa8011..9327d5e27 100644 --- a/src/test/regress/sql/hash_index_001.sql +++ b/src/test/regress/sql/hash_index_001.sql @@ -176,3 +176,28 @@ insert into hash_table_7 select random()*100, 'XXX', 'XXX' from generate_series( create index hash_t7_id1 on hash_table_7 using hash(id) with (fillfactor = 30); explain (costs off) select * from hash_table_7 where id = 80; select count(*) from hash_table_7; + +-- create hash index on segment table, update/delete +drop table if exists hash_table_8; +create table hash_table_8(id int, name varchar, sex varchar default 'male') with(segment = on); +create index hash_tb8_id1 on hash_table_8 using hash(id); +insert into hash_table_8 select generate_series(1, 1000), 'xxx', 'xxx'; +select count(*) from hash_table_8; +update hash_table_8 set sex = tmp.sex from (values (10, 'xxx', 'female'), (20, 'xxx', 'female'), (30, 'xxx', 'female')) as tmp (id, name, sex) where hash_table_8.id = tmp.id; +select * from hash_table_8 where sex = 'female'; +delete from hash_table_8 where sex = 'female'; +select * from hash_table_8 where sex = 'female'; +drop index hash_tb8_id1; +drop table hash_table_8; + +-- create hash index on segment table, delete/vacuum +drop table if exists hash_table_9; +create table hash_table_9(id int, name varchar, sec varchar default 'male') with (segment = on); +create index hash_tb9_id1 on hash_table_9 using hash(id); +insert into hash_table_9 select generate_series(1, 1000), 'XXX', 'XXX'; +insert into hash_table_9 select generate_series(1, 200), 'AAA', 'AAA'; +select count(*) from hash_table_9 where name = 'AAA'; +delete from hash_table_9 where name = 'AAA'; +select * from hash_table_9 where name = 'AAA'; +drop index hash_tb9_id1; +drop table hash_table_9; From cb23dfabb482835b8adae8038e62009bef65dc88 Mon Sep 17 00:00:00 2001 From: luozihao <1165977584@qq.com> Date: Thu, 7 Sep 2023 15:10:21 +0800 Subject: [PATCH 210/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=89=A9=E5=8C=96?= =?UTF-8?q?=E8=A7=86=E5=9B=BE=E5=88=B7=E6=96=B0=E6=9D=83=E9=99=90=E6=A3=80?= =?UTF-8?q?=E6=9F=A5=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../optimizer/commands/matview.cpp | 8 ++-- .../optimizer/commands/tablecmds.cpp | 45 +++++++++++++++++++ .../optimizer/rewrite/rewriteHandler.cpp | 2 +- src/include/commands/tablecmds.h | 2 + src/test/regress/expected/matview_single.out | 34 ++++++++++++++ src/test/regress/sql/matview_single.sql | 32 +++++++++++++ 6 files changed, 118 insertions(+), 5 deletions(-) diff --git a/src/gausskernel/optimizer/commands/matview.cpp b/src/gausskernel/optimizer/commands/matview.cpp index 25200d9c2..8d1c8064b 100755 --- a/src/gausskernel/optimizer/commands/matview.cpp +++ b/src/gausskernel/optimizer/commands/matview.cpp @@ -830,7 +830,7 @@ ObjectAddress ExecRefreshMatViewInc(RefreshMatViewStmt *stmt, const char *queryS matviewOid = RangeVarGetRelidExtended(stmt->relation, ExclusiveLock, false, false, false, true, - RangeVarCallbackOwnsTable, NULL); + RangeVarCallbackOwnsMatView, NULL); Oid mapid = DatumGetObjectId(get_matview_mapid(matviewOid)); Datum oldTime = get_matview_refreshtime(matviewOid, &isTimeNULL); @@ -946,7 +946,7 @@ ObjectAddress ExecRefreshIncMatViewAll(RefreshMatViewStmt *stmt, const char *que matviewOid = RangeVarGetRelidExtended(stmt->relation, AccessExclusiveLock, false, false, false, true, - RangeVarCallbackOwnsTable, NULL); + RangeVarCallbackOwnsMatView, NULL); mapid = DatumGetObjectId(get_matview_mapid(matviewOid)); matviewRel = heap_open(matviewOid, AccessExclusiveLock); @@ -1055,7 +1055,7 @@ ObjectAddress ExecRefreshCtasMatViewAll(RefreshMatViewStmt *stmt, const char *qu */ matviewOid = RangeVarGetRelidExtended(stmt->relation, AccessExclusiveLock, false, false, false, true, - RangeVarCallbackOwnsTable, NULL); + RangeVarCallbackOwnsMatView, NULL); matviewRel = heap_open(matviewOid, NoLock); /* @@ -1168,7 +1168,7 @@ bool isIncMatView(RangeVar *rv) Oid matviewOid = RangeVarGetRelidExtended(rv, NoLock, false, false, false, true, - RangeVarCallbackOwnsTable, NULL); + RangeVarCallbackOwnsMatView, NULL); Relation matviewRel = heap_open(matviewOid, AccessShareLock); /* Make sure it is a materialized view. */ diff --git a/src/gausskernel/optimizer/commands/tablecmds.cpp b/src/gausskernel/optimizer/commands/tablecmds.cpp index 337c62dd6..f5b9d406e 100755 --- a/src/gausskernel/optimizer/commands/tablecmds.cpp +++ b/src/gausskernel/optimizer/commands/tablecmds.cpp @@ -21920,6 +21920,51 @@ void RangeVarCallbackOwnsTable(const RangeVar* relation, Oid relId, Oid oldRelId } } +/* + * This is intended as a callback for RangeVarGetRelidExtended(). It allows + * the relation to be locked only if (1) it's a materialized view and + * (2) the current user is the owner (or the superuser). + * This meets the permission-checking needs of and REFRESH MATERIALIZED VIEW; + * we expose it here so that it can be used by all. + */ +void RangeVarCallbackOwnsMatView(const RangeVar* relation, Oid relId, Oid oldRelId, bool target_is_partition, void* arg) +{ + char relkind; + + /* Nothing to do if the relation was not found. */ + if (!OidIsValid(relId)) { + return; + } + + /* + * If the relation does exist, check whether it's an index. But note that + * the relation might have been dropped between the time we did the name + * lookup and now. In that case, there's nothing to do. + */ + relkind = get_rel_relkind(relId); + if (!relkind) { + return; + } + if (relkind != RELKIND_RELATION && + relkind != RELKIND_TOASTVALUE && + relkind != RELKIND_MATVIEW) { + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("\"%s\" is not a table or materialized view", relation->relname))); + } + + /* Check permissions */ + AclResult aclresult = pg_class_aclcheck(relId, GetUserId(), ACL_INSERT | ACL_DELETE); + if (aclresult != ACLCHECK_OK) { + aclcheck_error(aclresult, ACL_KIND_CLASS, relation->relname); + } + + bool is_owner = pg_class_ownercheck(relId, GetUserId()); + if (!is_owner) { + aclcheck_error(ACLCHECK_NOT_OWNER, ACL_KIND_CLASS, relation->relname); + } +} + /* * Callback to RangeVarGetRelidExtended(), similar to * RangeVarCallbackOwnsTable() but without checks on the type of the relation. diff --git a/src/gausskernel/optimizer/rewrite/rewriteHandler.cpp b/src/gausskernel/optimizer/rewrite/rewriteHandler.cpp index ece65fcf5..38073a378 100644 --- a/src/gausskernel/optimizer/rewrite/rewriteHandler.cpp +++ b/src/gausskernel/optimizer/rewrite/rewriteHandler.cpp @@ -4974,7 +4974,7 @@ List *QueryRewriteRefresh(Query *parse_tree) */ matviewOid = RangeVarGetRelidExtended(stmt->relation, AccessExclusiveLock, false, false, false, false, - RangeVarCallbackOwnsTable, NULL); + RangeVarCallbackOwnsMatView, NULL); matviewRel = heap_open(matviewOid, NoLock); /* Make sure it is a materialized view. */ diff --git a/src/include/commands/tablecmds.h b/src/include/commands/tablecmds.h index aa35104f9..b0a0339d3 100644 --- a/src/include/commands/tablecmds.h +++ b/src/include/commands/tablecmds.h @@ -181,6 +181,8 @@ extern void DropTableThrowErrorExternal(RangeVar* relation, ObjectType removeTyp extern void RangeVarCallbackOwnsTable( const RangeVar* relation, Oid relId, Oid oldRelId, bool target_is_partition, void* arg); +extern void RangeVarCallbackOwnsMatView( + const RangeVar* relation, Oid relId, Oid oldRelId, bool target_is_partition, void* arg); extern void RangeVarCallbackOwnsRelation( const RangeVar* relation, Oid relId, Oid oldRelId, bool target_is_partition, void* noCatalogs); extern void checkPartNotInUse(Partition part, const char* stmt); diff --git a/src/test/regress/expected/matview_single.out b/src/test/regress/expected/matview_single.out index 72131ba65..4d3c4042e 100644 --- a/src/test/regress/expected/matview_single.out +++ b/src/test/regress/expected/matview_single.out @@ -141,5 +141,39 @@ drop table test_syn cascade; NOTICE: drop cascades to 2 other objects DETAIL: drop cascades to materialized view mv_test_syn drop cascades to materialized view imv_test_syn +-- test about the privileges of refresh +create table t (id int); +insert into t select generate_series(1,10); +create materialized view mv_t as select * from t; +create user testuser with password 'Gauss@123'; +grant usage on schema public to testuser; +grant select on t to testuser; +grant select on mv_t to testuser; +set role testuser password 'Gauss@123'; +-- failed, permission denied +refresh materialized view mv_t; +ERROR: permission denied for relation mv_t +DETAIL: N/A +reset role; +grant delete,insert on mv_t to testuser; +set role testuser password 'Gauss@123'; +-- failed, permission denied +refresh materialized view mv_t; +ERROR: must be owner of relation mv_t +DETAIL: N/A +reset role; +grant index on mv_t to testuser; +set role testuser password 'Gauss@123'; +-- failed, permission denied +refresh materialized view mv_t; +ERROR: must be owner of relation mv_t +DETAIL: N/A +reset role; +alter table mv_t owner to testuser; +set role testuser password 'Gauss@123'; +-- success +refresh materialized view mv_t; +reset role; +drop user testuser cascade; \c regression drop database test_imv_db; diff --git a/src/test/regress/sql/matview_single.sql b/src/test/regress/sql/matview_single.sql index 56eca85d1..6f1502b9d 100644 --- a/src/test/regress/sql/matview_single.sql +++ b/src/test/regress/sql/matview_single.sql @@ -72,5 +72,37 @@ drop synonym s_mv_test_syn; drop synonym s_imv_test_syn; drop table test_syn cascade; +-- test about the privileges of refresh +create table t (id int); +insert into t select generate_series(1,10); +create materialized view mv_t as select * from t; +create user testuser with password 'Gauss@123'; +grant usage on schema public to testuser; +grant select on t to testuser; +grant select on mv_t to testuser; + +set role testuser password 'Gauss@123'; +-- failed, permission denied +refresh materialized view mv_t; +reset role; +grant delete,insert on mv_t to testuser; +set role testuser password 'Gauss@123'; +-- failed, permission denied +refresh materialized view mv_t; +reset role; +grant index on mv_t to testuser; +set role testuser password 'Gauss@123'; +-- failed, permission denied +refresh materialized view mv_t; + +reset role; +alter table mv_t owner to testuser; +set role testuser password 'Gauss@123'; +-- success +refresh materialized view mv_t; + +reset role; +drop user testuser cascade; + \c regression drop database test_imv_db; From 7e8f87435a6c6da452ef6f955fbe88809e5228f6 Mon Sep 17 00:00:00 2001 From: xuxinxin Date: Fri, 8 Sep 2023 16:20:27 +0800 Subject: [PATCH 211/304] =?UTF-8?q?3.0.5=E5=8D=87=E7=BA=A75.1.0=E5=A4=B1?= =?UTF-8?q?=E8=B4=A5=EF=BC=8C=E4=BF=AE=E6=94=B9ss=5Ftxnstatus=5Fcache=5Fst?= =?UTF-8?q?at=E4=B8=BA8889?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/catalog/builtin_funcs.ini | 2 +- .../upgrade-post_catalog_maindb_92_906.sql | 2 +- .../upgrade-post_catalog_otherdb_92_906.sql | 2 +- src/test/regress/expected/opr_sanity.out | 2 +- src/test/regress/expected/single_node_opr_sanity.out | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/common/backend/catalog/builtin_funcs.ini b/src/common/backend/catalog/builtin_funcs.ini index 77c2dc72f..13e1f3084 100755 --- a/src/common/backend/catalog/builtin_funcs.ini +++ b/src/common/backend/catalog/builtin_funcs.ini @@ -10989,7 +10989,7 @@ AddFuncGroup( ), AddFuncGroup( "ss_txnstatus_cache_stat", 1, - AddBuiltinFunc(_0(8888), _1("ss_txnstatus_cache_stat"), _2(0), _3(true), _4(true), _5(ss_txnstatus_cache_stat), _6(2249), _7(PG_CATALOG_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(1000), _12(0), _13(0), _14(false), _15(false), _16(false), _17(false), _18('v'), _19(0), _20(0), _21(8, 20, 20, 20, 701, 701, 701, 20, 701), _22(8, 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o'), _23(8, "vcache_gets", "hcache_gets", "nio_gets", "avg_hcache_gettime_us", "avg_nio_gettime_us", "cache_hit_rate", "hcache_eviction", "avg_eviction_refcnt"), _24(NULL), _25("ss_txnstatus_cache_stat"), _26(NULL), _27(NULL), _28(NULL), _29(0), _30(false), _31(NULL), _32(false), _33(NULL), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)) + AddBuiltinFunc(_0(8889), _1("ss_txnstatus_cache_stat"), _2(0), _3(true), _4(true), _5(ss_txnstatus_cache_stat), _6(2249), _7(PG_CATALOG_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(1000), _12(0), _13(0), _14(false), _15(false), _16(false), _17(false), _18('v'), _19(0), _20(0), _21(8, 20, 20, 20, 701, 701, 701, 20, 701), _22(8, 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o'), _23(8, "vcache_gets", "hcache_gets", "nio_gets", "avg_hcache_gettime_us", "avg_nio_gettime_us", "cache_hit_rate", "hcache_eviction", "avg_eviction_refcnt"), _24(NULL), _25("ss_txnstatus_cache_stat"), _26(NULL), _27(NULL), _28(NULL), _29(0), _30(false), _31(NULL), _32(false), _33(NULL), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)) ), AddFuncGroup( "start_collect_workload", 1, diff --git a/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_906.sql b/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_906.sql index 74f6fce90..71943ca07 100644 --- a/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_906.sql +++ b/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_906.sql @@ -12,7 +12,7 @@ BEGIN END$$; DROP FUNCTION IF EXISTS pg_catalog.ss_txnstatus_cache_stat() CASCADE; -SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 8888; +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 8889; CREATE FUNCTION pg_catalog.ss_txnstatus_cache_stat( OUT vcache_gets bigint, OUT hcache_gets bigint, diff --git a/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_906.sql b/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_906.sql index 74f6fce90..71943ca07 100644 --- a/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_906.sql +++ b/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_906.sql @@ -12,7 +12,7 @@ BEGIN END$$; DROP FUNCTION IF EXISTS pg_catalog.ss_txnstatus_cache_stat() CASCADE; -SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 8888; +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 8889; CREATE FUNCTION pg_catalog.ss_txnstatus_cache_stat( OUT vcache_gets bigint, OUT hcache_gets bigint, diff --git a/src/test/regress/expected/opr_sanity.out b/src/test/regress/expected/opr_sanity.out index bacd40ff8..7bd88ada1 100644 --- a/src/test/regress/expected/opr_sanity.out +++ b/src/test/regress/expected/opr_sanity.out @@ -2719,7 +2719,7 @@ WHERE d.classoid IS NULL AND p1.oid <= 9999 order by 1; 7777 | sysdate 7998 | set_working_grand_version_num_manually 8050 | datalength - 8888 | ss_txnstatus_cache_stat + 8889 | ss_txnstatus_cache_stat 9004 | smalldatetime_in 9006 | smalldatetime_out 9007 | smalldatetime_recv diff --git a/src/test/regress/expected/single_node_opr_sanity.out b/src/test/regress/expected/single_node_opr_sanity.out index 443d9c477..e8bfd2666 100755 --- a/src/test/regress/expected/single_node_opr_sanity.out +++ b/src/test/regress/expected/single_node_opr_sanity.out @@ -1552,7 +1552,7 @@ WHERE d.classoid IS NULL AND p1.oid <= 9999 order by 1; 8703 | array_except 8704 | array_except_distinct 8852 | pg_event_trigger_ddl_commands - 8888 | ss_txnstatus_cache_stat + 8889 | ss_txnstatus_cache_stat 9004 | smalldatetime_in 9006 | smalldatetime_out 9007 | smalldatetime_recv From 6b7d4c35bf0ad023628a04790454fa287dbbbf6f Mon Sep 17 00:00:00 2001 From: vastdata-xyzr Date: Fri, 8 Sep 2023 16:55:30 +0800 Subject: [PATCH 212/304] =?UTF-8?q?=E5=87=BD=E6=95=B0=20=5FequalCharsetCla?= =?UTF-8?q?use=20=E5=AE=9A=E4=B9=89=E5=BF=98=E8=AE=B0=E8=BF=94=E5=9B=9E?= =?UTF-8?q?=E5=80=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/nodes/equalfuncs.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/common/backend/nodes/equalfuncs.cpp b/src/common/backend/nodes/equalfuncs.cpp index 0395e0e99..505264be7 100644 --- a/src/common/backend/nodes/equalfuncs.cpp +++ b/src/common/backend/nodes/equalfuncs.cpp @@ -3486,6 +3486,7 @@ static bool _equalCharsetClause(const CharsetClause* a, const CharsetClause* b) COMPARE_SCALAR_FIELD(charset); COMPARE_SCALAR_FIELD(is_binary); COMPARE_LOCATION_FIELD(location); + return true; } static bool _equalPrefixKey(const PrefixKey* a, const PrefixKey* b) From 6578e4418985e21913582fd21c000af86c6df15b Mon Sep 17 00:00:00 2001 From: chenxiaobin19 <1025221611@qq.com> Date: Tue, 5 Sep 2023 17:59:56 +0800 Subject: [PATCH 213/304] =?UTF-8?q?=E6=94=AF=E6=8C=81=E8=87=AA=E5=8A=A8?= =?UTF-8?q?=E8=A1=A5=E9=BD=90=E8=AE=B0=E5=BD=95=E4=B9=8B=E5=89=8D=E8=A1=8C?= =?UTF-8?q?=E5=91=BD=E4=BB=A4=E4=B8=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/psql/input.cpp | 8 +++++++- src/bin/psql/input.h | 2 +- src/bin/psql/mainloop.cpp | 2 +- src/bin/psql/tab-complete.cpp | 36 ++++++++++++++++++++++++++++++++++- src/bin/psql/tab-complete.h | 2 ++ 5 files changed, 46 insertions(+), 4 deletions(-) diff --git a/src/bin/psql/input.cpp b/src/bin/psql/input.cpp index d6e51b988..8a927afd7 100644 --- a/src/bin/psql/input.cpp +++ b/src/bin/psql/input.cpp @@ -43,12 +43,15 @@ bool useReadline = false; * * Caller *must* have set up sigint_interrupt_jmp before calling. */ -char* gets_interactive(const char* prompt) +char* gets_interactive(const char* prompt, PQExpBuffer query_buf) { #ifdef USE_READLINE if (useReadline) { char* result = NULL; + /* Make current query_buf available to tab completion callback */ + tab_completion_query_buf = query_buf; + /* Enable SIGINT to longjmp to sigint_interrupt_jmp */ sigint_interrupt_enabled = true; @@ -58,6 +61,9 @@ char* gets_interactive(const char* prompt) /* Disable SIGINT again */ sigint_interrupt_enabled = false; + /* Pure neatnik-ism */ + tab_completion_query_buf = NULL; + return result; } #endif diff --git a/src/bin/psql/input.h b/src/bin/psql/input.h index 5e871518f..bf98e8df7 100644 --- a/src/bin/psql/input.h +++ b/src/bin/psql/input.h @@ -32,7 +32,7 @@ #include "libpq/pqexpbuffer.h" -char* gets_interactive(const char* prompt); +char* gets_interactive(const char* prompt, PQExpBuffer query_buf); char* gets_fromFile(FILE* source); void pg_append_history(const char* s, PQExpBuffer history_buf); diff --git a/src/bin/psql/mainloop.cpp b/src/bin/psql/mainloop.cpp index 73f59c468..4dbb728f6 100644 --- a/src/bin/psql/mainloop.cpp +++ b/src/bin/psql/mainloop.cpp @@ -408,7 +408,7 @@ int MainLoop(FILE* source, char* querystring) if (query_buf->len == 0) { prompt_status = PROMPT_READY; } - line = gets_interactive(get_prompt(prompt_status)); + line = gets_interactive(get_prompt(prompt_status), query_buf); } else { if (NULL != source) { /* fgets on SUSE12 may raise a buffer currupt of source->_IO_read_base. diff --git a/src/bin/psql/tab-complete.cpp b/src/bin/psql/tab-complete.cpp index 3e2d17836..df045ef3d 100644 --- a/src/bin/psql/tab-complete.cpp +++ b/src/bin/psql/tab-complete.cpp @@ -3559,10 +3559,33 @@ static PGresult *ExecQuery(const char *query) */ static void GetPreviousWords(int point, char **previousWords, int nwords) { - const char *buf = rl_line_buffer; /* alias */ + char *buf = NULL; /* alias */ int i; errno_t rc = EOK; + /* + * If we have anything in tab_completion_query_buf, paste it together with + * rl_line_buffer to construct the full query. Otherwise we can just use + * rl_line_buffer as the input string. + */ + if (tab_completion_query_buf && tab_completion_query_buf->len > 0) { + i = tab_completion_query_buf->len; + const int bufLen = point + i + 2; + buf = (char*)pg_malloc(bufLen); + rc = memcpy_s(buf, bufLen, tab_completion_query_buf->data, i); + securec_check_c(rc, "\0", "\0"); + + buf[i++] = '\n'; + rc = memcpy_s(buf + i, bufLen, rl_line_buffer, point); + securec_check_c(rc, "\0", "\0"); + i += point; + buf[i] = '\0'; + /* Readjust point to reference appropriate offset in buf */ + point = i; + } else { + buf = rl_line_buffer; + } + /* first we look for a non-word char before the current point */ for (i = point - 1; i >= 0; i--) if (strchr(WORD_BREAKS, buf[i])) @@ -3624,9 +3647,20 @@ static void GetPreviousWords(int point, char **previousWords, int nwords) *previousWords++ = s; } + + if (buf != rl_line_buffer) { + free(buf); + } } #endif /* HAVE_READLINE_READLINE_H */ +/* + * Since readline doesn't let us pass any state through to the tab completion + * callback, we have to use this global variable to let GetPreviousWords() + * get at the previous lines of the current command. + */ +PQExpBuffer tab_completion_query_buf = NULL; + /* * Initialize the readline library for our purposes. */ diff --git a/src/bin/psql/tab-complete.h b/src/bin/psql/tab-complete.h index 340606f02..90abea983 100644 --- a/src/bin/psql/tab-complete.h +++ b/src/bin/psql/tab-complete.h @@ -260,6 +260,8 @@ typedef struct { #endif /* HAVE_READLINE_READLINE_H */ +extern PQExpBuffer tab_completion_query_buf; + void initialize_readline(void); #endif From 96c6fd44d85a6546b00277b38c272e4d64bc09a6 Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Sat, 9 Sep 2023 15:46:16 +0800 Subject: [PATCH 214/304] =?UTF-8?q?dss=E6=8E=A8=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/ddes/ddes_commit_id | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gausskernel/ddes/ddes_commit_id b/src/gausskernel/ddes/ddes_commit_id index d22913055..c14e6068b 100644 --- a/src/gausskernel/ddes/ddes_commit_id +++ b/src/gausskernel/ddes/ddes_commit_id @@ -1,3 +1,3 @@ dms_commit_id=85eaf1d9de8d7486fdfa7a154405701bf294f490 -dss_commit_id=412bd233f74ffb84d2f9a715fd6d30450f56ff4c +dss_commit_id=87adc9bafa80354bb138a6bf9f8ff63077f3aebf cbb_commit_id=7a7e77f3dec94b6b958bea12c97b92d21dfa0bb5 \ No newline at end of file From 9eb463d3a7320ccb6d18ca633268ac522f3aca36 Mon Sep 17 00:00:00 2001 From: chendong76 <1209756284@qq.com> Date: Sun, 10 Sep 2023 14:17:12 +0800 Subject: [PATCH 215/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=9E=81=E8=87=B4RTO?= =?UTF-8?q?=E6=94=AF=E6=8C=81=E5=A4=87=E6=9C=BA=E8=AF=BB=E7=89=B9=E6=80=A7?= =?UTF-8?q?=E5=AF=B9=E6=8C=89=E9=9C=80=E5=9B=9E=E6=94=BE=E7=89=B9=E6=80=A7?= =?UTF-8?q?=E7=9A=84=E5=BD=B1=E5=93=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../storage/access/redo/redo_xlogutils.cpp | 2 +- .../access/transam/extreme_rto_redo_api.cpp | 10 -------- .../ondemand_extreme_rto/dispatcher.cpp | 2 +- .../storage/access/transam/xlog.cpp | 2 +- src/gausskernel/storage/buffer/bufmgr.cpp | 2 +- src/gausskernel/storage/ipc/procarray.cpp | 2 +- src/include/access/extreme_rto_redo_api.h | 24 +++++++++++++++++-- src/include/access/multi_redo_api.h | 2 +- 8 files changed, 28 insertions(+), 18 deletions(-) diff --git a/src/gausskernel/storage/access/redo/redo_xlogutils.cpp b/src/gausskernel/storage/access/redo/redo_xlogutils.cpp index 7bfcbaf22..cf6feafb7 100644 --- a/src/gausskernel/storage/access/redo/redo_xlogutils.cpp +++ b/src/gausskernel/storage/access/redo/redo_xlogutils.cpp @@ -1768,7 +1768,7 @@ bool XLogBlockRedoForExtremeRTO(XLogRecParseState *redoblocktate, RedoBufferInfo } if ((block_valid != BLOCK_DATA_UNDO_TYPE) && g_instance.attr.attr_storage.EnableHotStandby && - XLByteLT(PageGetLSN(bufferinfo->pageinfo.page), blockhead->end_ptr)) { + IsDefaultExtremeRtoMode() && XLByteLT(PageGetLSN(bufferinfo->pageinfo.page), blockhead->end_ptr)) { BufferTag buf_tag; INIT_BUFFERTAG(buf_tag, bufferinfo->blockinfo.rnode, bufferinfo->blockinfo.forknum, bufferinfo->blockinfo.blkno); diff --git a/src/gausskernel/storage/access/transam/extreme_rto_redo_api.cpp b/src/gausskernel/storage/access/transam/extreme_rto_redo_api.cpp index f8c7eddea..84f6d328e 100644 --- a/src/gausskernel/storage/access/transam/extreme_rto_redo_api.cpp +++ b/src/gausskernel/storage/access/transam/extreme_rto_redo_api.cpp @@ -141,16 +141,6 @@ static const f_extreme_rto_redo extreme_rto_redosw[] = { }, }; -void SetExtremeRtoMode() -{ - g_extreme_rto_type = DEFAULT_EXTREME_RTO; -} - -void SetOndemandExtremeRtoMode() -{ - g_extreme_rto_type = ONDEMAND_EXTREME_RTO; -} - void ExtremeWaitAllReplayWorkerIdle() { (*(extreme_rto_redosw[g_extreme_rto_type].wait_all_replay_worker_idle))(); diff --git a/src/gausskernel/storage/access/transam/ondemand_extreme_rto/dispatcher.cpp b/src/gausskernel/storage/access/transam/ondemand_extreme_rto/dispatcher.cpp index cab1ab125..9e8e0336c 100644 --- a/src/gausskernel/storage/access/transam/ondemand_extreme_rto/dispatcher.cpp +++ b/src/gausskernel/storage/access/transam/ondemand_extreme_rto/dispatcher.cpp @@ -1874,7 +1874,7 @@ void SendRecoveryEndMarkToWorkersAndWaitForReach(int code) } } } - ereport(LOG, (errmsg("[SS][REDO_LOG_TRACE] lastReadXact: %lu, trxnComplete: %lu, pageMgrComplele: %lu", + ereport(DEBUG1, (errmsg("[SS][REDO_LOG_TRACE] lastReadXact: %lu, trxnComplete: %lu, pageMgrComplele: %lu", lastReadEndPtr, trxnCompletePtr, pageMngrCompletePtr))); if (XLByteEQ(trxnCompletePtr, lastReadEndPtr) && XLByteEQ(pageMngrCompletePtr, lastReadEndPtr)) { break; diff --git a/src/gausskernel/storage/access/transam/xlog.cpp b/src/gausskernel/storage/access/transam/xlog.cpp index 13e63c2c9..66c6aba57 100755 --- a/src/gausskernel/storage/access/transam/xlog.cpp +++ b/src/gausskernel/storage/access/transam/xlog.cpp @@ -9622,7 +9622,7 @@ void StartupXLOG(void) } g_instance.dms_cxt.SSRecoveryInfo.in_ondemand_recovery = false; - SetExtremeRtoMode(); + SetDefaultExtremeRtoMode(); if (SS_PRIMARY_MODE && ENABLE_ONDEMAND_RECOVERY && (SS_STANDBY_FAILOVER || SS_PRIMARY_NORMAL_REFORM) && t_thrd.xlog_cxt.InRecovery == true) { if (SSOndemandRecoveryExitNormal) { diff --git a/src/gausskernel/storage/buffer/bufmgr.cpp b/src/gausskernel/storage/buffer/bufmgr.cpp index 63cf83f6f..c52978bda 100644 --- a/src/gausskernel/storage/buffer/bufmgr.cpp +++ b/src/gausskernel/storage/buffer/bufmgr.cpp @@ -1673,7 +1673,7 @@ Buffer ReadBuffer(Relation reln, BlockNumber block_num) Buffer ReadBufferExtended(Relation reln, ForkNumber fork_num, BlockNumber block_num, ReadBufferMode mode, BufferAccessStrategy strategy) { - if (IsExtremeRtoRunning() && !AmPageRedoWorker()) { + if (IsDefaultExtremeRtoMode() && IsExtremeRtoRunning() && !AmPageRedoWorker()) { return standby_read_buf(reln, fork_num, block_num, mode, strategy); } diff --git a/src/gausskernel/storage/ipc/procarray.cpp b/src/gausskernel/storage/ipc/procarray.cpp index 39e78271b..b1bd477c9 100755 --- a/src/gausskernel/storage/ipc/procarray.cpp +++ b/src/gausskernel/storage/ipc/procarray.cpp @@ -2435,7 +2435,7 @@ Snapshot GetSnapshotData(Snapshot snapshot, bool force_local_snapshot) (void)pgstat_report_waitstatus(oldStatus); } - if (IsExtremeRtoRunning() && pmState == PM_HOT_STANDBY) { + if (IsDefaultExtremeRtoMode() && IsExtremeRtoRunning() && pmState == PM_HOT_STANDBY) { extreme_rto::exrto_read_snapshot(snapshot); } diff --git a/src/include/access/extreme_rto_redo_api.h b/src/include/access/extreme_rto_redo_api.h index ef86785bf..13df44646 100644 --- a/src/include/access/extreme_rto_redo_api.h +++ b/src/include/access/extreme_rto_redo_api.h @@ -36,8 +36,28 @@ typedef enum { extern ExtremeRtoRedoType g_extreme_rto_type; -void SetExtremeRtoMode(); -void SetOndemandExtremeRtoMode(); +inline void SetDefaultExtremeRtoMode() +{ + g_extreme_rto_type = DEFAULT_EXTREME_RTO; +} + +inline void SetOndemandExtremeRtoMode() +{ + g_extreme_rto_type = ONDEMAND_EXTREME_RTO; +} + +inline bool IsDefaultExtremeRtoMode() +{ + return (g_extreme_rto_type == DEFAULT_EXTREME_RTO); +} + +inline bool IsOndemandExtremeRtoMode() +{ + return (g_extreme_rto_type == ONDEMAND_EXTREME_RTO); +} +inline void SetOndemandExtremeRtoMode(); +inline bool IsDefaultExtremeRtoMode(); +inline bool IsOndemandExtremeRtoMode(); void ExtremeWaitAllReplayWorkerIdle(); void ExtremeDispatchCleanInvalidPageMarkToAllRedoWorker(RepairFileKey key); void ExtremeDispatchClosefdMarkToAllRedoWorker(); diff --git a/src/include/access/multi_redo_api.h b/src/include/access/multi_redo_api.h index 331bcd897..3dbc82dca 100644 --- a/src/include/access/multi_redo_api.h +++ b/src/include/access/multi_redo_api.h @@ -66,7 +66,7 @@ static const uint32 PAGE_REDO_WORKER_READY = 2; static const uint32 PAGE_REDO_WORKER_EXIT = 3; static const uint32 BIG_RECORD_LENGTH = XLOG_BLCKSZ * 16; -#define IS_EXRTO_READ (g_instance.attr.attr_storage.EnableHotStandby && IsExtremeRedo()) +#define IS_EXRTO_READ (g_instance.attr.attr_storage.EnableHotStandby && IsExtremeRedo() && IsDefaultExtremeRtoMode()) #define IS_EXRTO_STANDBY_READ (IS_EXRTO_READ && pm_state_is_hot_standby()) static inline int get_real_recovery_parallelism() From 02434606b550f1ab17a00b7315af2e2ec9c49d88 Mon Sep 17 00:00:00 2001 From: shenzheng4 Date: Mon, 11 Sep 2023 10:22:40 +0800 Subject: [PATCH 216/304] gs_ctl check add help --- src/bin/pg_ctl/pg_ctl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bin/pg_ctl/pg_ctl.cpp b/src/bin/pg_ctl/pg_ctl.cpp index 57bf63823..91a1cfc79 100755 --- a/src/bin/pg_ctl/pg_ctl.cpp +++ b/src/bin/pg_ctl/pg_ctl.cpp @@ -3771,7 +3771,7 @@ static void do_help(void) printf(_(" -b, --mode=MODE the mode of building the datanode or coordinator." "MODE can be \"full\", \"incremental\", " "\"auto\", \"standby_full\", \"copy_secure_files\", \"copy_upgrade_file\", \"cross_cluster_full\", " - "\"cross_cluster_incremental\", \"cross_cluster_standby_full\"\n")); + "\"cross_cluster_incremental\", \"cross_cluster_standby_full\", \"check\"\n")); printf(_(" -D, --pgdata=DATADIR location of the database storage area\n")); printf(_(" -s, --silent only print errors, no informational messages\n")); printf(_(" -t, --timeout=SECS seconds to wait when using -w option\n")); From 139bde8612738e3a42a36d01af0a5ef6d551e9e4 Mon Sep 17 00:00:00 2001 From: dongning12 Date: Mon, 11 Sep 2023 10:23:44 +0800 Subject: [PATCH 217/304] =?UTF-8?q?=E3=80=90=E8=B5=84=E6=BA=90=E6=B1=A0?= =?UTF-8?q?=E5=8C=96=E3=80=91owner=E8=BD=AC=E7=A7=BB=E9=A1=B5=E9=9D=A2?= =?UTF-8?q?=E5=9C=BA=E6=99=AF=EF=BC=8C=E5=AF=B9=E4=BA=8Ebeen=5Fload?= =?UTF-8?q?=E4=B8=BAfalse=E6=83=85=E5=86=B5=EF=BC=8C=E8=A1=A5=E5=85=85?= =?UTF-8?q?=E5=86=85=E5=AE=B9=E9=94=81=E7=9A=84=E9=87=8A=E6=94=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/ddes/adapter/ss_dms_callback.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/gausskernel/ddes/adapter/ss_dms_callback.cpp b/src/gausskernel/ddes/adapter/ss_dms_callback.cpp index c0af89cf8..4adc3cd60 100644 --- a/src/gausskernel/ddes/adapter/ss_dms_callback.cpp +++ b/src/gausskernel/ddes/adapter/ss_dms_callback.cpp @@ -605,6 +605,7 @@ static int tryEnterLocalPage(BufferTag *tag, dms_lock_mode_t mode, dms_buf_ctrl_ Assert(buf_id >= 0); if ((*buf_ctrl)->been_loaded == false) { *buf_ctrl = NULL; + LWLockRelease(buf_desc->content_lock); DmsReleaseBuffer(buf_desc->buf_id + 1, is_seg); ereport(WARNING, (errmodule(MOD_DMS), errmsg("[%u/%u/%u/%d %d-%u] been_loaded marked false, page swapped out and failed to load", From 99d6590e94ca013801b60ef674aa928dc53c36f7 Mon Sep 17 00:00:00 2001 From: z00848344 Date: Mon, 11 Sep 2023 15:11:00 +0800 Subject: [PATCH 218/304] On branch zym-hash-segment Your branch is up to date with 'origin/zym-hash-segment'. Changes to be committed: modified: src/gausskernel/storage/access/hash/hashpage.cpp --- src/gausskernel/storage/access/hash/hashpage.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gausskernel/storage/access/hash/hashpage.cpp b/src/gausskernel/storage/access/hash/hashpage.cpp index aa96262c1..c8baba488 100644 --- a/src/gausskernel/storage/access/hash/hashpage.cpp +++ b/src/gausskernel/storage/access/hash/hashpage.cpp @@ -996,7 +996,7 @@ static bool _hash_alloc_buckets(Relation rel, BlockNumber firstblock, uint32 nbl if (IsSegmentFileNode(rel->rd_node)) { Buffer buf; - for (int i = firstblock; i <= lastblock; i++) { + for (BlockNumber i = firstblock; i <= lastblock; i++) { buf = ReadBuffer(rel, P_NEW); ReleaseBuffer(buf); } From 0e24f499adfb05a194b217d21cb4e6452e2ca064 Mon Sep 17 00:00:00 2001 From: dongning12 Date: Mon, 11 Sep 2023 20:35:42 +0800 Subject: [PATCH 219/304] =?UTF-8?q?=E3=80=90=E8=B5=84=E6=BA=90=E6=B1=A0?= =?UTF-8?q?=E5=8C=96=E3=80=91=E6=9B=B4=E6=96=B0DMS=E7=9A=84commit=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/ddes/adapter/ss_dms.cpp | 4 +- .../ddes/adapter/ss_dms_bufmgr.cpp | 2 +- src/gausskernel/ddes/ddes_commit_id | 2 +- src/include/ddes/dms/dms_api.h | 60 ++++++++++++++----- src/include/ddes/dms/ss_dms.h | 4 +- 5 files changed, 50 insertions(+), 22 deletions(-) diff --git a/src/gausskernel/ddes/adapter/ss_dms.cpp b/src/gausskernel/ddes/adapter/ss_dms.cpp index ee759b06d..b95ef62e5 100644 --- a/src/gausskernel/ddes/adapter/ss_dms.cpp +++ b/src/gausskernel/ddes/adapter/ss_dms.cpp @@ -307,9 +307,9 @@ int dms_get_ssl_param(const char *param_name, char *param_value, unsigned int si return g_ss_dms_func.dms_get_ssl_param(param_name, param_value, size); } -int dms_recovery_page_need_skip(char pageid[DMS_PAGEID_SIZE], unsigned char *skip) +int dms_recovery_page_need_skip(char pageid[DMS_PAGEID_SIZE], unsigned char *skip, unsigned int alloc) { - return g_ss_dms_func.dms_recovery_page_need_skip(pageid, skip); + return g_ss_dms_func.dms_recovery_page_need_skip(pageid, skip, alloc); } int dms_reform_failed(void) diff --git a/src/gausskernel/ddes/adapter/ss_dms_bufmgr.cpp b/src/gausskernel/ddes/adapter/ss_dms_bufmgr.cpp index 7bea64e42..97656b595 100644 --- a/src/gausskernel/ddes/adapter/ss_dms_bufmgr.cpp +++ b/src/gausskernel/ddes/adapter/ss_dms_bufmgr.cpp @@ -724,7 +724,7 @@ bool CheckPageNeedSkipInRecovery(Buffer buf) char pageid[DMS_PAGEID_SIZE]; errno_t err = memcpy_s(pageid, DMS_PAGEID_SIZE, &(buf_desc->tag), sizeof(BufferTag)); securec_check(err, "\0", "\0"); - int ret = dms_recovery_page_need_skip(pageid, (unsigned char *)&skip); + int ret = dms_recovery_page_need_skip(pageid, (unsigned char *)&skip, false); if (ret != DMS_SUCCESS) { ereport(PANIC, (errmsg("DMS Internal error happened during recovery, errno %d", ret))); } diff --git a/src/gausskernel/ddes/ddes_commit_id b/src/gausskernel/ddes/ddes_commit_id index c14e6068b..811136323 100644 --- a/src/gausskernel/ddes/ddes_commit_id +++ b/src/gausskernel/ddes/ddes_commit_id @@ -1,3 +1,3 @@ -dms_commit_id=85eaf1d9de8d7486fdfa7a154405701bf294f490 +dms_commit_id=d7095d9b169a766461b00aa4bc0e19b4fc8ea657 dss_commit_id=87adc9bafa80354bb138a6bf9f8ff63077f3aebf cbb_commit_id=7a7e77f3dec94b6b958bea12c97b92d21dfa0bb5 \ No newline at end of file diff --git a/src/include/ddes/dms/dms_api.h b/src/include/ddes/dms/dms_api.h index 8aa775b9b..03bd86b2d 100644 --- a/src/include/ddes/dms/dms_api.h +++ b/src/include/ddes/dms/dms_api.h @@ -442,12 +442,6 @@ typedef struct st_dw_recovery_info { unsigned long long bitmap_new_join; // the new-join-inst bitmap in dw_recovery phase } dw_recovery_info_t; -typedef struct st_file_orglsn_recovery_info { - unsigned long long bitmap_old_join; // the old-join-inst bitmap in dw_recovery phase - unsigned long long bitmap_old_remove; // the old-remove-inst bitmap in dw_recovery phase - unsigned long long bitmap_new_join; // the new-join-inst bitmap in dw_recovery phase -} file_orglsn_recovery_info_t; - typedef struct st_inst_list { unsigned char inst_id_list[DMS_MAX_INSTANCES]; unsigned char inst_id_count; @@ -553,6 +547,39 @@ typedef enum en_dms_inst_behavior { DMS_INST_BEHAVIOR_IN_BACKUP, } dms_inst_behavior_t; +typedef struct st_stat_buf_info { + int instance_id; + unsigned char lock_mode; /* which lock held by instance of this buffer */ + unsigned long int mem_lsn; /* page lsn in memory */ + unsigned long int rec_lsn; /* recovery LSN */ + unsigned long int lsn_on_disk; /* page lsn in dick */ + unsigned long int dirty_queue_loc; /* actual location of dirty page queue */ + char aio_in_progress; /* indicate aio is in progress */ + char data[DMS_RESID_SIZE]; /* user defined resource(page) identifier */ +} stat_buf_info_t; +/* +* used by openGauss server to get DRC information +*/ +typedef struct st_stat_drc_info { + stat_buf_info_t buf_info[DMS_MAX_INSTANCES]; /* save buffer related information */ + dms_context_t dms_ctx; + unsigned char master_id; + unsigned long long copy_insts; /* bitmap for owners, for S mode, more than one owner may exist */ + unsigned char claimed_owner; /* owner */ + unsigned char lock_mode; /* current DRC lock mode */ + unsigned char last_edp; /* the newest edp instance id */ + unsigned char type; /* page or lock */ + unsigned char in_recovery; /* in recovery or not */ + unsigned char copy_promote; /* copy promote to owner, can not release, may need flush */ + unsigned short part_id; /* which partition id that current page belongs to */ + unsigned long long edp_map; /* indicate which instance has current page's EDP(Earlier Dirty Page) */ + unsigned long long lsn; /* the newest edp LSN of current page in the cluster */ + unsigned short len; /* the length of data below */ + unsigned char recovery_skip; /* DRC is accessed in recovery and skip because drc has owner */ + unsigned char recycling; + char data[DMS_RESID_SIZE]; /* user defined resource(page) identifier */ +} stat_drc_info_t; + typedef int(*dms_get_list_stable)(void *db_handle, unsigned long long *list_stable, unsigned char *reformer_id); typedef int(*dms_save_list_stable)(void *db_handle, unsigned long long list_stable, unsigned char reformer_id, unsigned long long list_in, unsigned int save_ctrl); @@ -569,7 +596,6 @@ typedef int(*dms_disk_lsn)(void *db_handle, char *pageid, unsigned long long *ls typedef int(*dms_recovery)(void *db_handle, void *recovery_list, int is_reformer); typedef int(*dms_dw_recovery)(void *db_handle, void *recovery_list, int is_reformer); typedef int(*dms_df_recovery)(void *db_handle); -typedef int(*dms_file_orglsn_recovery)(void *db_handle, void *recovery_list, int is_reformer); typedef int(*dms_opengauss_startup)(void *db_handle); typedef int(*dms_opengauss_recovery_standby)(void *db_handle, int inst_id); typedef int(*dms_opengauss_recovery_primary)(void *db_handle, int inst_id); @@ -711,6 +737,7 @@ typedef int (*dms_verify_page_checksum)(void *db_handle, dms_buf_ctrl_t *ctrl, u typedef int (*dms_update_node_oldest_xmin)(void *db_handle, unsigned char inst_id, unsigned long long oldest_xmin); typedef void (*dms_set_inst_behavior)(void *db_handle, dms_inst_behavior_t inst_behavior); typedef int (*dms_db_prepare)(void *db_handle); +typedef void (*dms_get_buf_info)(char* resid, stat_buf_info_t *buf_info); typedef struct st_dms_callback { // used in reform @@ -725,10 +752,9 @@ typedef struct st_dms_callback { dms_edp_lsn edp_lsn; dms_disk_lsn disk_lsn; dms_recovery recovery; + dms_recovery recovery_analyse; dms_dw_recovery dw_recovery; dms_df_recovery df_recovery; - dms_file_orglsn_recovery file_orglsn_recovery_part1; - dms_file_orglsn_recovery file_orglsn_recovery_part2; dms_db_is_primary db_is_primary; dms_get_open_status get_open_status; dms_undo_init undo_init; @@ -856,6 +882,8 @@ typedef struct st_dms_callback { //for shared storage backup dms_set_inst_behavior set_inst_behavior; dms_db_prepare db_prepare; + + dms_get_buf_info get_buf_info; } dms_callback_t; typedef struct st_dms_instance_net_addr { @@ -927,19 +955,19 @@ typedef enum en_dms_info_id { DMS_INFO_REFORM_LAST = 1, } dms_info_id_e; -typedef enum st_protocol_version { - PROTO_VER_0 = 0, // invalid version - PROTO_VER_1 = 1, // first version -} protocol_version_e; +typedef enum st_dms_protocol_version { + DMS_PROTO_VER_0 = 0, // invalid version + DMS_PROTO_VER_1 = 1, // first version +} dms_protocol_version_e; -#define INVALID_PROTO_VER PROTO_VER_0 -#define SW_PROTO_VER PROTO_VER_1 +#define DMS_INVALID_PROTO_VER DMS_PROTO_VER_0 +#define DMS_SW_PROTO_VER DMS_PROTO_VER_1 #define DMS_LOCAL_MAJOR_VER_WEIGHT 1000000 #define DMS_LOCAL_MINOR_VER_WEIGHT 1000 #define DMS_LOCAL_MAJOR_VERSION 0 #define DMS_LOCAL_MINOR_VERSION 0 -#define DMS_LOCAL_VERSION 89 +#define DMS_LOCAL_VERSION 92 #ifdef __cplusplus } diff --git a/src/include/ddes/dms/ss_dms.h b/src/include/ddes/dms/ss_dms.h index 1bf0d8fda..77ca5148e 100644 --- a/src/include/ddes/dms/ss_dms.h +++ b/src/include/ddes/dms/ss_dms.h @@ -65,7 +65,7 @@ typedef struct st_ss_dms_func { int (*dms_register_ssl_decrypt_pwd)(dms_decrypt_pwd_t cb_func); int (*dms_set_ssl_param)(const char *param_name, const char *param_value); int (*dms_get_ssl_param)(const char *param_name, char *param_value, unsigned int size); - int (*dms_recovery_page_need_skip)(char pageid[DMS_PAGEID_SIZE], unsigned char *skip); + int (*dms_recovery_page_need_skip)(char pageid[DMS_PAGEID_SIZE], unsigned char *skip, unsigned int alloc); int (*dms_reform_failed)(void); int (*dms_switchover)(unsigned int sess_id); int (*dms_drc_accessible)(unsigned char res_type); @@ -117,7 +117,7 @@ int drc_get_page_master_id(char pageid[DMS_PAGEID_SIZE], unsigned char *master_i int dms_register_ssl_decrypt_pwd(dms_decrypt_pwd_t cb_func); int dms_set_ssl_param(const char *param_name, const char *param_value); int dms_get_ssl_param(const char *param_name, char *param_value, unsigned int size); -int dms_recovery_page_need_skip(char pageid[DMS_PAGEID_SIZE], unsigned char *skip); +int dms_recovery_page_need_skip(char pageid[DMS_PAGEID_SIZE], unsigned char *skip, unsigned int alloc); int dms_reform_failed(void); int dms_switchover(unsigned int sess_id); int dms_drc_accessible(unsigned char res_type); From 4f0f0f1245fa2a5c2db995f8625b53f2f1f97a9e Mon Sep 17 00:00:00 2001 From: l30039603 Date: Tue, 12 Sep 2023 10:42:14 +0800 Subject: [PATCH 220/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=B5=8C=E5=A5=97?= =?UTF-8?q?=E5=87=BD=E6=95=B0=E6=96=AD=E7=82=B9=E5=81=9C=E7=95=99=E4=BD=8D?= =?UTF-8?q?=E7=BD=AE=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/pl/plpgsql/src/pl_debugger.cpp | 2 +- .../regress/expected/pl_debugger_client.out | 18 ++++++++++-------- src/test/regress/sql/pl_debugger_client.sql | 3 ++- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/common/pl/plpgsql/src/pl_debugger.cpp b/src/common/pl/plpgsql/src/pl_debugger.cpp index d9d7269f7..d2676b8b5 100644 --- a/src/common/pl/plpgsql/src/pl_debugger.cpp +++ b/src/common/pl/plpgsql/src/pl_debugger.cpp @@ -190,7 +190,7 @@ void check_debug(PLpgSQL_function* func, PLpgSQL_execstate* estate) /* maintain session's debug server is on base turn on function */ u_sess->plsql_cxt.cur_debug_server = func->debug; } - func->debug->stop_next_stmt = true; + func->debug->stop_next_stmt = need_continue_into ? false : true; } } diff --git a/src/test/regress/expected/pl_debugger_client.out b/src/test/regress/expected/pl_debugger_client.out index 4c9d95ff2..9d994e05a 100755 --- a/src/test/regress/expected/pl_debugger_client.out +++ b/src/test/regress/expected/pl_debugger_client.out @@ -1559,22 +1559,23 @@ select funcname, lineno, query from dbe_pldebugger.continue(); "anonymous block" | 8 | k = test_increment(3); (1 row) -select funcname, lineno, query from dbe_pldebugger.step(); +insert into tmp_holder select * from dbe_pldebugger.add_breakpoint((select oid from pg_proc where proname='test_increment'), 6); -- ok +select funcname, lineno, query from dbe_pldebugger.continue(); funcname | lineno | query ----------------+--------+----------------- - test_increment | 5 | x := x + 1; + test_increment | 6 | y := y * 2; (1 row) -select funcname, lineno, query from dbe_pldebugger.next(); - funcname | lineno | query -----------------+--------+----------------- - test_increment | 6 | y := y * 2; +select funcname, lineno, query from dbe_pldebugger.step(); + funcname | lineno | query +----------------+--------+--------------- + test_increment | 7 | RETURN y; (1 row) select frameno, funcname, lineno, query from dbe_pldebugger.backtrace(); frameno | funcname | lineno | query ---------+-------------------+--------+---------------------------- - 0 | test_increment | 6 | y := y * 2; + 0 | test_increment | 7 | RETURN y; 1 | "anonymous block" | 8 | k = test_increment(3); (2 rows) @@ -1614,5 +1615,6 @@ select * from tmp_holder; true true true -(8 rows) + 2 +(9 rows) diff --git a/src/test/regress/sql/pl_debugger_client.sql b/src/test/regress/sql/pl_debugger_client.sql index aca8125d5..1e2873fe8 100644 --- a/src/test/regress/sql/pl_debugger_client.sql +++ b/src/test/regress/sql/pl_debugger_client.sql @@ -483,8 +483,9 @@ insert into tmp_holder select * from dbe_pldebugger.enable_breakpoint(1); -- ok insert into tmp_holder select * from dbe_pldebugger.delete_breakpoint(0); -- ok select funcname, lineno, query from dbe_pldebugger.next(); select funcname, lineno, query from dbe_pldebugger.continue(); +insert into tmp_holder select * from dbe_pldebugger.add_breakpoint((select oid from pg_proc where proname='test_increment'), 6); -- ok +select funcname, lineno, query from dbe_pldebugger.continue(); select funcname, lineno, query from dbe_pldebugger.step(); -select funcname, lineno, query from dbe_pldebugger.next(); select frameno, funcname, lineno, query from dbe_pldebugger.backtrace(); select * from dbe_pldebugger.info_locals(); select funcname, lineno, query from dbe_pldebugger.finish(); From a0d66679411ff2e8c96cee7d0d2907660371adb5 Mon Sep 17 00:00:00 2001 From: totaj Date: Wed, 6 Sep 2023 10:39:07 +0800 Subject: [PATCH 221/304] Record query's parse tree nodetag for dolphin. userd to check query's type, for strict mode. --- src/common/backend/catalog/pg_proc.cpp | 5 ++-- src/common/backend/parser/analyze.cpp | 1 + src/common/pl/plpgsql/src/pl_comp.cpp | 4 ++++ src/common/pl/plpgsql/src/pl_exec.cpp | 18 +++++++++++--- src/common/pl/plpgsql/src/pl_handler.cpp | 7 +++++- .../optimizer/commands/prepare.cpp | 5 +++- src/gausskernel/process/stream/streamMain.cpp | 2 ++ src/gausskernel/process/tcop/postgres.cpp | 24 +++++++++++++++++++ .../process/threadpool/knl_thread.cpp | 1 + .../process/threadpool/threadpool_stream.cpp | 1 + .../runtime/executor/functions.cpp | 6 +++++ src/gausskernel/runtime/executor/spi.cpp | 12 ++++++++++ src/include/knl/knl_thread.h | 1 + src/include/nodes/parsenodes.h | 19 +++++++++++++++ 14 files changed, 99 insertions(+), 7 deletions(-) diff --git a/src/common/backend/catalog/pg_proc.cpp b/src/common/backend/catalog/pg_proc.cpp index 5ca671a7a..14356df94 100644 --- a/src/common/backend/catalog/pg_proc.cpp +++ b/src/common/backend/catalog/pg_proc.cpp @@ -1963,7 +1963,7 @@ Datum fmgr_sql_validator(PG_FUNCTION_ARGS) ErrorContextCallback sqlerrcontext; bool haspolyarg = false; int i; - + NodeTag old_node_tag = t_thrd.postgres_cxt.cur_command_tag; bool replace = false; /* * 3 means the number of arguments of function fmgr_sql_validator, while 'is_replace' is the third one, @@ -2049,7 +2049,7 @@ Datum fmgr_sql_validator(PG_FUNCTION_ARGS) foreach (lc, raw_parsetree_list) { Node* parsetree = (Node*)lfirst(lc); List* querytree_sublist = NIL; - + t_thrd.postgres_cxt.cur_command_tag = transform_node_tag(parsetree); #ifdef PGXC /* Block CTAS in SQL functions */ if (IsA(parsetree, CreateTableAsStmt)) @@ -2093,6 +2093,7 @@ Datum fmgr_sql_validator(PG_FUNCTION_ARGS) } ReleaseSysCache(tuple); + t_thrd.postgres_cxt.cur_command_tag = old_node_tag; PG_RETURN_VOID(); } diff --git a/src/common/backend/parser/analyze.cpp b/src/common/backend/parser/analyze.cpp index 12c7763ec..b00bca043 100644 --- a/src/common/backend/parser/analyze.cpp +++ b/src/common/backend/parser/analyze.cpp @@ -4569,6 +4569,7 @@ static Query* transformExplainStmt(ParseState* pstate, ExplainStmt* stmt) { Query* result = NULL; + t_thrd.postgres_cxt.cur_command_tag = transform_node_tag(stmt->query); /* transform contained query, allowing SELECT INTO */ stmt->query = (Node*)transformTopLevelStmt(pstate, stmt->query); diff --git a/src/common/pl/plpgsql/src/pl_comp.cpp b/src/common/pl/plpgsql/src/pl_comp.cpp index 8b76146e4..8255d637a 100644 --- a/src/common/pl/plpgsql/src/pl_comp.cpp +++ b/src/common/pl/plpgsql/src/pl_comp.cpp @@ -4934,6 +4934,7 @@ TupleDesc getCursorTupleDesc(PLpgSQL_expr* expr, bool isOnlySelect, bool isOnlyP expr->func->datums = u_sess->plsql_cxt.curr_compile_context->plpgsql_Datums; expr->func->ndatums = u_sess->plsql_cxt.curr_compile_context->plpgsql_nDatums; TupleDesc tupleDesc = NULL; + NodeTag old_node_tag = t_thrd.postgres_cxt.cur_command_tag; PG_TRY(); { List* parsetreeList = pg_parse_query(expr->query); @@ -4945,6 +4946,7 @@ TupleDesc getCursorTupleDesc(PLpgSQL_expr* expr, bool isOnlySelect, bool isOnlyP List* queryList = NIL; foreach(cell, parsetreeList) { Node *parsetree = (Node *)lfirst(cell); + t_thrd.postgres_cxt.cur_command_tag = transform_node_tag(parsetree); if (nodeTag(parsetree) == T_SelectStmt) { if (checkSelectIntoParse((SelectStmt*)parsetree)) { list_free_deep(parsetreeList); @@ -4991,6 +4993,7 @@ TupleDesc getCursorTupleDesc(PLpgSQL_expr* expr, bool isOnlySelect, bool isOnlyP } PG_CATCH(); { + t_thrd.postgres_cxt.cur_command_tag = old_node_tag; /* Save error info */ MemoryContext ecxt = MemoryContextSwitchTo(current_context); ErrorData* edata = CopyErrorData(); @@ -5011,6 +5014,7 @@ TupleDesc getCursorTupleDesc(PLpgSQL_expr* expr, bool isOnlySelect, bool isOnlyP } PG_END_TRY(); + t_thrd.postgres_cxt.cur_command_tag = old_node_tag; return tupleDesc; } static int get_inner_type_ind(Oid typeoid) diff --git a/src/common/pl/plpgsql/src/pl_exec.cpp b/src/common/pl/plpgsql/src/pl_exec.cpp index fde5e6e68..ccbd9c319 100644 --- a/src/common/pl/plpgsql/src/pl_exec.cpp +++ b/src/common/pl/plpgsql/src/pl_exec.cpp @@ -1303,6 +1303,7 @@ Datum plpgsql_exec_function(PLpgSQL_function* func, FunctionCallInfo fcinfo, boo } #endif + NodeTag old_node_tag = t_thrd.postgres_cxt.cur_command_tag; saved_current_stp_with_exception = plpgsql_get_current_value_stp_with_exception(); /* * Setup error traceback support for ereport() @@ -1532,7 +1533,7 @@ Datum plpgsql_exec_function(PLpgSQL_function* func, FunctionCallInfo fcinfo, boo estate.err_text = gettext_noop("while casting return value to function's return type"); fcinfo->isnull = estate.retisnull; - + t_thrd.postgres_cxt.cur_command_tag = T_CreateStmt; if (estate.retisset) { ReturnSetInfo* rsi = estate.rsi; @@ -1743,7 +1744,7 @@ Datum plpgsql_exec_function(PLpgSQL_function* func, FunctionCallInfo fcinfo, boo /* Clean up any leftover temporary memory */ plpgsql_destroy_econtext(&estate); exec_eval_cleanup(&estate); - + t_thrd.postgres_cxt.cur_command_tag = old_node_tag; /* * Pop the error context stack */ @@ -3915,6 +3916,7 @@ static int exec_stmt(PLpgSQL_execstate* estate, PLpgSQL_stmt* stmt, bool resigna { PLpgSQL_stmt* save_estmt = NULL; int rc = -1; + NodeTag old_command_tag; save_estmt = estate->err_stmt; estate->err_stmt = stmt; @@ -3949,12 +3951,14 @@ static int exec_stmt(PLpgSQL_execstate* estate, PLpgSQL_stmt* stmt, bool resigna } } + old_command_tag = t_thrd.postgres_cxt.cur_command_tag; switch ((enum PLpgSQL_stmt_types)stmt->cmd_type) { case PLPGSQL_STMT_BLOCK: rc = exec_stmt_block(estate, (PLpgSQL_stmt_block*)stmt, NULL, resignal_in_handler); break; case PLPGSQL_STMT_ASSIGN: + t_thrd.postgres_cxt.cur_command_tag = T_CreateStmt; rc = exec_stmt_assign(estate, (PLpgSQL_stmt_assign*)stmt); break; @@ -4011,14 +4015,17 @@ static int exec_stmt(PLpgSQL_execstate* estate, PLpgSQL_stmt* stmt, bool resigna break; case PLPGSQL_STMT_RETURN: + t_thrd.postgres_cxt.cur_command_tag = T_CreateStmt; rc = exec_stmt_return(estate, (PLpgSQL_stmt_return*)stmt); break; case PLPGSQL_STMT_RETURN_NEXT: + t_thrd.postgres_cxt.cur_command_tag = T_CreateStmt; rc = exec_stmt_return_next(estate, (PLpgSQL_stmt_return_next*)stmt); break; case PLPGSQL_STMT_RETURN_QUERY: + t_thrd.postgres_cxt.cur_command_tag = T_SelectStmt; rc = exec_stmt_return_query(estate, (PLpgSQL_stmt_return_query*)stmt); break; @@ -4058,6 +4065,7 @@ static int exec_stmt(PLpgSQL_execstate* estate, PLpgSQL_stmt* stmt, bool resigna } case PLPGSQL_STMT_FETCH: + t_thrd.postgres_cxt.cur_command_tag = T_CreateStmt; rc = exec_stmt_fetch(estate, (PLpgSQL_stmt_fetch*)stmt); break; @@ -4095,6 +4103,7 @@ static int exec_stmt(PLpgSQL_execstate* estate, PLpgSQL_stmt* stmt, bool resigna errmsg("unrecognized statement type: %d for PLSQL function.", stmt->cmd_type))); break; } + t_thrd.postgres_cxt.cur_command_tag = old_command_tag; /* Let the plugin know that we have finished executing this statement */ if (*u_sess->plsql_cxt.plugin_ptr && (*u_sess->plsql_cxt.plugin_ptr)->stmt_end) { @@ -6796,7 +6805,7 @@ static int exec_stmt_execsql(PLpgSQL_execstate* estate, PLpgSQL_stmt_execsql* st Cursor_Data* saved_cursor_data = NULL; bool has_alloc = false; bool multi_res_return = false; - + NodeTag old_node_tag = t_thrd.postgres_cxt.cur_command_tag; TransactionId oldTransactionId = SPI_get_top_transaction_id(); /* @@ -6826,6 +6835,8 @@ static int exec_stmt_execsql(PLpgSQL_execstate* estate, PLpgSQL_stmt_execsql* st } } } + + t_thrd.postgres_cxt.cur_command_tag = stmt->mod_stmt ? T_CreateSetStmt : T_SelectStmt; if (ENABLE_CN_GPC && g_instance.plan_cache->CheckRecreateSPICachePlan(expr->plan)) { g_instance.plan_cache->RecreateSPICachePlan(expr->plan); } @@ -7105,6 +7116,7 @@ static int exec_stmt_execsql(PLpgSQL_execstate* estate, PLpgSQL_stmt_execsql* st estate->cursor_return_data = saved_cursor_data; estate->cursor_return_numbers = saved_cursor_numbers; + t_thrd.postgres_cxt.cur_command_tag = old_node_tag; return PLPGSQL_RC_OK; } diff --git a/src/common/pl/plpgsql/src/pl_handler.cpp b/src/common/pl/plpgsql/src/pl_handler.cpp index 3d1967f08..1e0cf297c 100755 --- a/src/common/pl/plpgsql/src/pl_handler.cpp +++ b/src/common/pl/plpgsql/src/pl_handler.cpp @@ -808,6 +808,7 @@ Datum plpgsql_call_handler(PG_FUNCTION_ARGS) /* Must save and restore prior value of cur_estate and debug_info */ save_cur_estate = func->cur_estate; save_debug_info = func->debug; + NodeTag old_node_tag = t_thrd.postgres_cxt.cur_command_tag; // set the procedure's search_path as the current search_path validate_search_path(func); @@ -879,7 +880,7 @@ Datum plpgsql_call_handler(PG_FUNCTION_ARGS) u_sess->plsql_cxt.cur_exception_cxt = NULL; t_thrd.log_cxt.call_stack = saveplcallstack; - + t_thrd.postgres_cxt.cur_command_tag = old_node_tag; #ifndef ENABLE_MULTIPLE_NODES /* for restore parent session and automn session package var values */ @@ -957,6 +958,7 @@ Datum plpgsql_call_handler(PG_FUNCTION_ARGS) DecreasePackageUseCount(func); func->cur_estate = save_cur_estate; func->debug = save_debug_info; + t_thrd.postgres_cxt.cur_command_tag = old_node_tag; // resume the search_path when the procedure has executed PopOverrideSearchPath(); @@ -1127,6 +1129,7 @@ Datum plpgsql_inline_handler(PG_FUNCTION_ARGS) int save_compile_list_length = list_length(u_sess->plsql_cxt.compile_context_list); int save_compile_status = u_sess->plsql_cxt.compile_status; DebugInfo* save_debug_info = func->debug; + NodeTag old_node_tag = t_thrd.postgres_cxt.cur_command_tag; FormatCallStack* saveplcallstack = t_thrd.log_cxt.call_stack; PG_TRY(); { @@ -1163,6 +1166,7 @@ Datum plpgsql_inline_handler(PG_FUNCTION_ARGS) dopControl.ResetSmp(); #endif + t_thrd.postgres_cxt.cur_command_tag = old_node_tag; ereport(DEBUG3, (errmodule(MOD_NEST_COMPILE), errcode(ERRCODE_LOG), errmsg("%s clear curr_compile_context because of error.", __func__))); /* reset nest plpgsql compile */ @@ -1190,6 +1194,7 @@ Datum plpgsql_inline_handler(PG_FUNCTION_ARGS) } func->debug = save_debug_info; #endif + t_thrd.postgres_cxt.cur_command_tag = old_node_tag; if (u_sess->SPI_cxt._connected == 0) { t_thrd.utils_cxt.STPSavedResourceOwner = NULL; } diff --git a/src/gausskernel/optimizer/commands/prepare.cpp b/src/gausskernel/optimizer/commands/prepare.cpp index cd0c556d7..ca252acc6 100755 --- a/src/gausskernel/optimizer/commands/prepare.cpp +++ b/src/gausskernel/optimizer/commands/prepare.cpp @@ -164,6 +164,7 @@ void PrepareQuery(PrepareStmt* stmt, const char* queryString) stmt->name, #endif CreateCommandTag(stmt->query)); + t_thrd.postgres_cxt.cur_command_tag = transform_node_tag(stmt->query); /* Transform list of TypeNames to array of type OIDs */ nargs = list_length(stmt->argtypes); @@ -314,6 +315,7 @@ void ExecuteQuery(ExecuteStmt* stmt, IntoClause* intoClause, const char* querySt /* Look it up in the hash table */ entry = FetchPreparedStatement(stmt->name, true, true); psrc = entry->plansource; + t_thrd.postgres_cxt.cur_command_tag = transform_node_tag(psrc->raw_parse_tree); /* Shouldn't find a non-fixed-result cached plan */ if (!entry->plansource->fixed_result) @@ -1761,6 +1763,7 @@ void RePrepareQuery(ExecuteStmt* stmt) */ foreach (parsetree_item, parseTree_list) { Node* parsetree = (Node*)lfirst(parsetree_item); + t_thrd.postgres_cxt.cur_command_tag = transform_node_tag(parsetree); List* planTree_list = NIL; queryTree_list = pg_analyze_and_rewrite(parsetree, query_string, NULL, 0); @@ -2152,4 +2155,4 @@ bool quickPlanner(List* querytree_list, Node* parsetree, const char*queryString, if (estate != NULL) FreeExecutorState(estate); return true; -} \ No newline at end of file +} diff --git a/src/gausskernel/process/stream/streamMain.cpp b/src/gausskernel/process/stream/streamMain.cpp index e5d1bb041..222e894b8 100755 --- a/src/gausskernel/process/stream/streamMain.cpp +++ b/src/gausskernel/process/stream/streamMain.cpp @@ -141,6 +141,7 @@ int StreamMain() WLMReleaseIoInfoFromHash(); /* Reset here so that we can get debug_query_string when Stream thread is in Sync point */ t_thrd.postgres_cxt.debug_query_string = NULL; + t_thrd.postgres_cxt.cur_command_tag = T_Invalid; /* * Note that parent thread will do commit or abort transaction. @@ -440,6 +441,7 @@ static void execute_stream_plan(StreamProducer* producer) // For now plan shipping is used only for SELECTs, in future // we should remove this hard coding and get the tag automatically commandTag = "SELECT"; + t_thrd.postgres_cxt.cur_command_tag = T_SelectStmt; set_ps_display(commandTag, false); diff --git a/src/gausskernel/process/tcop/postgres.cpp b/src/gausskernel/process/tcop/postgres.cpp index b670a3a6a..b08806c83 100755 --- a/src/gausskernel/process/tcop/postgres.cpp +++ b/src/gausskernel/process/tcop/postgres.cpp @@ -2675,6 +2675,7 @@ static void exec_simple_query(const char* query_string, MessageType messageType, * destination. */ commandTag = CreateCommandTag(parsetree); + t_thrd.postgres_cxt.cur_command_tag = transform_node_tag(parsetree); set_ps_display(commandTag, false); if (libpqsw_skip_check_readonly()) { @@ -3107,6 +3108,7 @@ static void exec_simple_query(const char* query_string, MessageType messageType, TRACE_POSTGRESQL_QUERY_DONE(query_string); t_thrd.postgres_cxt.debug_query_string = NULL; + t_thrd.postgres_cxt.cur_command_tag = T_Invalid; /* * @hdfs @@ -3694,6 +3696,7 @@ static void exec_parse_message(const char* query_string, /* string to execute */ * Get the command name for possible use in status display. */ commandTag = CreateCommandTag(raw_parse_tree); + t_thrd.postgres_cxt.cur_command_tag = transform_node_tag(raw_parse_tree); /* * If we are in an aborted transaction, reject all commands except @@ -3968,6 +3971,7 @@ static void exec_parse_message(const char* query_string, /* string to execute */ ShowUsage("PARSE MESSAGE STATISTICS"); t_thrd.postgres_cxt.debug_query_string = NULL; + t_thrd.postgres_cxt.cur_command_tag = T_Invalid; gstrace_exit(GS_TRC_ID_exec_parse_message); } @@ -4566,6 +4570,7 @@ static void exec_bind_message(StringInfo input_message) * Report query to various monitoring facilities. */ t_thrd.postgres_cxt.debug_query_string = psrc->query_string; + t_thrd.postgres_cxt.cur_command_tag = transform_node_tag(psrc->raw_parse_tree); pgstat_report_activity(STATE_RUNNING, psrc->query_string); instr_stmt_report_start_time(); @@ -5143,6 +5148,7 @@ static void exec_bind_message(StringInfo input_message) ShowUsage("BIND MESSAGE STATISTICS"); t_thrd.postgres_cxt.debug_query_string = NULL; + t_thrd.postgres_cxt.cur_command_tag = T_Invalid; gstrace_exit(GS_TRC_ID_exec_bind_message); } @@ -5244,6 +5250,16 @@ static void exec_execute_message(const char* portal_name, long max_rows) * Report query to various monitoring facilities. */ t_thrd.postgres_cxt.debug_query_string = sourceText; + if (strncmp(portal->commandTag, "SELECT", strlen("SELECT")) == 0) { + /* SELECT INTO is select stmt too */ + t_thrd.postgres_cxt.cur_command_tag = T_SelectStmt; + } else if (strcmp(portal->commandTag, "SHOW") == 0) { + t_thrd.postgres_cxt.cur_command_tag = T_VariableShowStmt; + } else if (strcmp(portal->commandTag, "CALL") == 0) { + t_thrd.postgres_cxt.cur_command_tag = T_DolphinCallStmt; + } else { + t_thrd.postgres_cxt.cur_command_tag = T_CreateStmt; + } pgstat_report_activity(STATE_RUNNING, sourceText); @@ -5435,6 +5451,7 @@ static void exec_execute_message(const char* portal_name, long max_rows) ShowUsage("EXECUTE MESSAGE STATISTICS"); t_thrd.postgres_cxt.debug_query_string = NULL; + t_thrd.postgres_cxt.cur_command_tag = T_Invalid; gstrace_exit(GS_TRC_ID_exec_execute_message); } @@ -8400,6 +8417,7 @@ int PostgresMain(int argc, char* argv[], const char* dbname, const char* usernam * the storage it points at. */ t_thrd.postgres_cxt.debug_query_string = NULL; + t_thrd.postgres_cxt.cur_command_tag = T_Invalid; if (u_sess->unique_sql_cxt.need_update_calls && is_unique_sql_enabled() && is_local_unique_sql()) { @@ -8550,6 +8568,7 @@ int PostgresMain(int argc, char* argv[], const char* dbname, const char* usernam initStringInfo(&input_message); t_thrd.postgres_cxt.debug_query_string = NULL; + t_thrd.postgres_cxt.cur_command_tag = T_Invalid; t_thrd.postgres_cxt.g_NoAnalyzeRelNameList = NIL; u_sess->analyze_cxt.is_under_analyze = false; u_sess->exec_cxt.isLockRows = false; @@ -9561,6 +9580,7 @@ int PostgresMain(int argc, char* argv[], const char* dbname, const char* usernam } pfree_ext(completionTag); t_thrd.postgres_cxt.debug_query_string = NULL; + t_thrd.postgres_cxt.cur_command_tag = T_Invalid; if (MEMORY_TRACKING_QUERY_PEAK) ereport(LOG, (errmsg("execute opfusion, peak memory %ld(kb)", (int64)(t_thrd.utils_cxt.peakedBytesInQueryLifeCycle/1024)))); @@ -10838,6 +10858,7 @@ static void exec_one_in_batch(CachedPlanSource* psrc, ParamListInfo params, int } #endif + t_thrd.postgres_cxt.cur_command_tag = transform_node_tag(psrc->raw_parse_tree); if (psrc->opFusionObj != NULL) { (void)RevalidateCachedQuery(psrc); OpFusion *opFusionObj = (OpFusion *)(psrc->opFusionObj); @@ -11002,6 +11023,7 @@ static void exec_one_in_batch(CachedPlanSource* psrc, ParamListInfo params, int psrc->stmt_name, psrc->query_string))); } + t_thrd.postgres_cxt.cur_command_tag = T_Invalid; } /* @@ -11461,6 +11483,7 @@ static void exec_batch_bind_execute(StringInfo input_message) * Report query to various monitoring facilities. */ t_thrd.postgres_cxt.debug_query_string = psrc->query_string; + t_thrd.postgres_cxt.cur_command_tag = transform_node_tag(psrc->raw_parse_tree); pgstat_report_activity(STATE_RUNNING, psrc->query_string); set_ps_display(psrc->commandTag, false); @@ -12101,6 +12124,7 @@ static void exec_batch_bind_execute(StringInfo input_message) } t_thrd.postgres_cxt.debug_query_string = NULL; + t_thrd.postgres_cxt.cur_command_tag = T_Invalid; } /* lock function for g_instance.codegen_IRload_process_count Addition */ diff --git a/src/gausskernel/process/threadpool/knl_thread.cpp b/src/gausskernel/process/threadpool/knl_thread.cpp index 3c26794b7..4a604a239 100755 --- a/src/gausskernel/process/threadpool/knl_thread.cpp +++ b/src/gausskernel/process/threadpool/knl_thread.cpp @@ -850,6 +850,7 @@ static void knl_t_postgres_init(knl_t_postgres_context* postgres_cxt) { postgres_cxt->clear_key_memory = false; postgres_cxt->debug_query_string = NULL; + postgres_cxt->cur_command_tag = T_Invalid; postgres_cxt->isInResetUserName = false; postgres_cxt->whereToSendOutput = DestDebug; postgres_cxt->local_foreign_respool = NULL; diff --git a/src/gausskernel/process/threadpool/threadpool_stream.cpp b/src/gausskernel/process/threadpool/threadpool_stream.cpp index 5ba6d6284..22012f599 100644 --- a/src/gausskernel/process/threadpool/threadpool_stream.cpp +++ b/src/gausskernel/process/threadpool/threadpool_stream.cpp @@ -182,6 +182,7 @@ static void ResetStreamStatus() /* Add the pg_delete_audit operation to audit log */ t_thrd.audit.Audit_delete = false; t_thrd.postgres_cxt.debug_query_string = NULL; + t_thrd.postgres_cxt.cur_command_tag = T_Invalid; t_thrd.postgres_cxt.g_NoAnalyzeRelNameList = NIL; t_thrd.postgres_cxt.mark_explain_analyze = false; t_thrd.postgres_cxt.mark_explain_only = false; diff --git a/src/gausskernel/runtime/executor/functions.cpp b/src/gausskernel/runtime/executor/functions.cpp index 7fc99bd9b..98c043b2e 100644 --- a/src/gausskernel/runtime/executor/functions.cpp +++ b/src/gausskernel/runtime/executor/functions.cpp @@ -589,6 +589,7 @@ static void init_sql_fcache(FmgrInfo* finfo, Oid collation, bool lazy_eval_ok) ListCell* lc = NULL; Datum tmp; bool is_null = false; + NodeTag old_node_tag = t_thrd.postgres_cxt.cur_command_tag; /* * Create memory context that holds all the SQLFunctionCache data. It @@ -704,6 +705,7 @@ static void init_sql_fcache(FmgrInfo* finfo, Oid collation, bool lazy_eval_ok) foreach (lc, raw_parsetree_list) { Node* parsetree = (Node*)lfirst(lc); List* queryTree_sublist = NIL; + t_thrd.postgres_cxt.cur_command_tag = transform_node_tag(parsetree); queryTree_sublist = pg_analyze_and_rewrite_params(parsetree, fcache->src, (ParserSetupHook)sql_fn_parser_setup, fcache->pinfo); @@ -751,6 +753,7 @@ static void init_sql_fcache(FmgrInfo* finfo, Oid collation, bool lazy_eval_ok) /* Mark fcache with time of creation to show it's valid */ fcache->lxid = t_thrd.proc->lxid; fcache->subxid = GetCurrentSubTransactionId(); + t_thrd.postgres_cxt.cur_command_tag = old_node_tag; ReleaseSysCache(procedure_tuple); @@ -1052,6 +1055,7 @@ Datum fmgr_sql(PG_FUNCTION_ARGS) bool old_running_in_fmgr = t_thrd.codegen_cxt.g_runningInFmgr; t_thrd.codegen_cxt.g_runningInFmgr = true; bool need_snapshot = !ActiveSnapshotSet(); + NodeTag old_node_tag = t_thrd.postgres_cxt.cur_command_tag; #ifdef ENABLE_MULTIPLE_NODES bool outer_is_stream = false; @@ -1211,6 +1215,7 @@ Datum fmgr_sql(PG_FUNCTION_ARGS) pushed_snapshot = true; } + t_thrd.postgres_cxt.cur_command_tag = es->qd->operation == CMD_SELECT ? T_SelectStmt : T_CreateStmt; completed = postquel_getnext(es, fcache); /* * If we ran the command to completion, we can shut it down now. Any @@ -1411,6 +1416,7 @@ Datum fmgr_sql(PG_FUNCTION_ARGS) u_sess->opt_cxt.query_dop = outerDop; #endif t_thrd.codegen_cxt.g_runningInFmgr = old_running_in_fmgr; + t_thrd.postgres_cxt.cur_command_tag = old_node_tag; return result; } diff --git a/src/gausskernel/runtime/executor/spi.cpp b/src/gausskernel/runtime/executor/spi.cpp index 6c76256e8..76c7f4111 100644 --- a/src/gausskernel/runtime/executor/spi.cpp +++ b/src/gausskernel/runtime/executor/spi.cpp @@ -1646,6 +1646,7 @@ static Portal SPI_cursor_open_internal(const char *name, SPIPlanPtr plan, ParamL AutoDopControl dopControl; dopControl.CloseSmp(); #endif + NodeTag old_node_tag = t_thrd.postgres_cxt.cur_command_tag; /* * Check that the plan is something the Portal code will special-case as @@ -1666,6 +1667,7 @@ static Portal SPI_cursor_open_internal(const char *name, SPIPlanPtr plan, ParamL Assert(list_length(plan->plancache_list) == 1); plansource = (CachedPlanSource *)linitial(plan->plancache_list); + t_thrd.postgres_cxt.cur_command_tag = transform_node_tag(plansource->raw_parse_tree); SPI_STACK_LOG("begin", NULL, plan); /* Push the SPI stack */ @@ -1830,6 +1832,7 @@ static Portal SPI_cursor_open_internal(const char *name, SPIPlanPtr plan, ParamL /* reset flag */ u_sess->SPI_cxt.has_stream_in_cursor_or_forloop_sql = false; + t_thrd.postgres_cxt.cur_command_tag = old_node_tag; _SPI_end_call(true); @@ -2300,6 +2303,7 @@ static void _SPI_pgxc_prepare_plan(const char *src, List *src_parsetree, SPIPlan spi_err_context.arg = (void *)src; spi_err_context.previous = t_thrd.log_cxt.error_context_stack; t_thrd.log_cxt.error_context_stack = &spi_err_context; + NodeTag old_node_tag = t_thrd.postgres_cxt.cur_command_tag; /* * Parse the request string into a list of raw parse trees. @@ -2322,6 +2326,7 @@ static void _SPI_pgxc_prepare_plan(const char *src, List *src_parsetree, SPIPlan Node *parsetree = (Node *)lfirst(list_item); List *stmt_list = NIL; CachedPlanSource *plansource = NULL; + t_thrd.postgres_cxt.cur_command_tag = transform_node_tag(parsetree); // get cachedplan if has any enable_spi_gpc = false; if (ENABLE_CN_GPC && u_sess->SPI_cxt._current->spi_hash_key != INVALID_SPI_KEY @@ -2413,6 +2418,7 @@ static void _SPI_pgxc_prepare_plan(const char *src, List *src_parsetree, SPIPlan plan->plancache_list = plancache_list; plan->oneshot = false; u_sess->SPI_cxt._current->plan_id = -1; + t_thrd.postgres_cxt.cur_command_tag = old_node_tag; /* * Pop the error context stack @@ -2471,6 +2477,7 @@ void _SPI_prepare_oneshot_plan(const char *src, SPIPlanPtr plan, parse_query_fun ErrorContextCallback spi_err_context; List *query_string_locationlist = NIL; int stmt_num = 0; + NodeTag old_node_tag = t_thrd.postgres_cxt.cur_command_tag; /* * Setup error traceback support for ereport() */ @@ -2493,6 +2500,7 @@ void _SPI_prepare_oneshot_plan(const char *src, SPIPlanPtr plan, parse_query_fun foreach (list_item, raw_parsetree_list) { Node *parsetree = (Node *)lfirst(list_item); CachedPlanSource *plansource = NULL; + t_thrd.postgres_cxt.cur_command_tag = transform_node_tag(parsetree); #ifndef ENABLE_MULTIPLE_NODES if (g_instance.attr.attr_sql.enableRemoteExcute) { @@ -2518,6 +2526,7 @@ void _SPI_prepare_oneshot_plan(const char *src, SPIPlanPtr plan, parse_query_fun plan->plancache_list = plancache_list; plan->oneshot = true; + t_thrd.postgres_cxt.cur_command_tag = old_node_tag; /* * Pop the error context stack @@ -2630,6 +2639,7 @@ static int _SPI_execute_plan0(SPIPlanPtr plan, ParamListInfo paramLI, Snapshot s CachedPlan *cplan = NULL; ListCell *lc1 = NULL; bool tmp_enable_light_proxy = u_sess->attr.attr_sql.enable_light_proxy; + NodeTag old_command_tag = t_thrd.postgres_cxt.cur_command_tag; TransactionId oldTransactionId = SPI_get_top_transaction_id(); bool need_remember_cplan = false; @@ -2690,6 +2700,7 @@ static int _SPI_execute_plan0(SPIPlanPtr plan, ParamListInfo paramLI, Snapshot s List *stmt_list = NIL; spi_err_context.arg = (void *)plansource->query_string; + t_thrd.postgres_cxt.cur_command_tag = transform_node_tag(plansource->raw_parse_tree); /* * If this is a one-shot plan, we still need to do parse analysis. @@ -2980,6 +2991,7 @@ static int _SPI_execute_plan0(SPIPlanPtr plan, ParamListInfo paramLI, Snapshot s } u_sess->attr.attr_sql.enable_light_proxy = tmp_enable_light_proxy; + t_thrd.postgres_cxt.cur_command_tag = old_command_tag; return my_res; } diff --git a/src/include/knl/knl_thread.h b/src/include/knl/knl_thread.h index 088c655bb..593420494 100755 --- a/src/include/knl/knl_thread.h +++ b/src/include/knl/knl_thread.h @@ -1695,6 +1695,7 @@ typedef struct knl_t_postgres_context { bool clear_key_memory; const char* debug_query_string; /* client-supplied query string */ + NodeTag cur_command_tag; /* current execute sql nodetag */ bool isInResetUserName; /* Note: whereToSendOutput is initialized for the bootstrap/standalone case */ diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index fe7e86ae9..0482e009c 100755 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -2385,5 +2385,24 @@ typedef struct GetDiagStmt { bool hasCondNum; List *condNum; } GetDiagStmt; + +extern inline NodeTag transform_node_tag(Node* raw_parse_tree) +{ + if (!raw_parse_tree) { + return T_Invalid; + } + if (nodeTag(raw_parse_tree) == T_SelectStmt) { + SelectStmt *stmt = (SelectStmt *)raw_parse_tree; + /* treat select into @var and select into file as common select */ + if (stmt->intoClause == NULL || stmt->intoClause->userVarList != NIL || stmt->intoClause->filename != NULL) { + return T_SelectStmt; + } + return T_CreateStmt; + } else if (nodeTag(raw_parse_tree) == T_ExplainStmt) { + return transform_node_tag(((ExplainStmt*)raw_parse_tree)->query); + } + return nodeTag(raw_parse_tree); +} + #endif /* PARSENODES_H */ From 52b3b4b189cd9681b896af3130ab024d83b84e4c Mon Sep 17 00:00:00 2001 From: wuyuechuan Date: Tue, 12 Sep 2023 16:42:50 +0800 Subject: [PATCH 222/304] disable 'table of type' as column in tables --- src/common/backend/parser/parse_utilcmd.cpp | 43 +++++++++++++++++++ .../regress/expected/tableof_unsupported.out | 16 +++++++ src/test/regress/sql/tableof_unsupported.sql | 12 ++++++ 3 files changed, 71 insertions(+) create mode 100644 src/test/regress/expected/tableof_unsupported.out create mode 100644 src/test/regress/sql/tableof_unsupported.sql diff --git a/src/common/backend/parser/parse_utilcmd.cpp b/src/common/backend/parser/parse_utilcmd.cpp index 7c3ac0e17..ab7aa98a6 100644 --- a/src/common/backend/parser/parse_utilcmd.cpp +++ b/src/common/backend/parser/parse_utilcmd.cpp @@ -5283,6 +5283,48 @@ static void transformConstraintAttrs(CreateStmtContext* cxt, List* constraintLis } } +/** + * tableof type is not supported be a column in a table + * @param ctype ctype + */ +static void CheckColumnTableOfType(Type ctype) +{ + Form_pg_type typTup = (Form_pg_type)GETSTRUCT(ctype); + if (typTup->typtype == TYPTYPE_TABLEOF) { + ereport(ERROR, (errcode(ERRCODE_DATATYPE_MISMATCH), errmodule(MOD_PLSQL), + errmsg("type \"%s\" is not supported as column type", NameStr(typTup->typname)), + errdetail("\"%s\" is a nest table type", NameStr(typTup->typname)), + errcause("feature not supported"), erraction("check type name"))); + } else if (typTup->typtype == TYPTYPE_COMPOSITE) { + TupleDesc tupleDesc = lookup_rowtype_tupdesc_noerror(HeapTupleGetOid(ctype), typTup->typtypmod, true); + if (tupleDesc == NULL) { + ereport(ERROR, (errcode(ERRCODE_CACHE_LOOKUP_FAILED), + errmsg("type %u cannot get tupledesc", HeapTupleGetOid(ctype)))); + } + for (int i = 0; i < tupleDesc->natts; i++) { + if (tupleDesc->attrs[i].attisdropped) { + continue; + } + HeapTuple typeTuple = SearchSysCache1(TYPEOID, ObjectIdGetDatum(tupleDesc->attrs[i].atttypid)); + if (!HeapTupleIsValid(typeTuple)) { + ereport(ERROR, (errcode(ERRCODE_CACHE_LOOKUP_FAILED), + errmsg("cache lookup failed for type %u", tupleDesc->attrs[i].atttypid))); + } + CheckColumnTableOfType(typeTuple); + ReleaseSysCache(typeTuple); + } + ReleaseTupleDesc(tupleDesc); + } else if (OidIsValid(typTup->typelem) && typTup->typtype == TYPTYPE_BASE) { + HeapTuple typTuple = SearchSysCache1(TYPEOID, ObjectIdGetDatum(typTup->typelem)); + if (!HeapTupleIsValid(typTuple)) { + ereport(ERROR, + (errcode(ERRCODE_CACHE_LOOKUP_FAILED), errmsg("cache lookup failed for type %u", typTup->typelem))); + } + CheckColumnTableOfType(typTuple); + ReleaseSysCache(typTuple); + } +} + /* * Special handling of type definition for a column */ @@ -5324,6 +5366,7 @@ static void transformColumnType(CreateStmtContext* cxt, ColumnDef* column) erraction("check type name"))); } #endif + CheckColumnTableOfType(ctype); ReleaseSysCache(ctype); } diff --git a/src/test/regress/expected/tableof_unsupported.out b/src/test/regress/expected/tableof_unsupported.out new file mode 100644 index 000000000..9cfc8a93e --- /dev/null +++ b/src/test/regress/expected/tableof_unsupported.out @@ -0,0 +1,16 @@ +------ Prepare ------ +-- create type of TYPTYPE_TABLEOF +create type r0 is (c1 int1, c2 int2); +create type t0 is table of int4; +create type t1 is (c1 int1, c2 t0); +create type t2 is table of r0; +-- create table +create table tableof_unsupported(id t0); +ERROR: type "t0" is not supported as column type +DETAIL: "t0" is a nest table type +create table tableof_unsupported(id t1); +ERROR: type "t0" is not supported as column type +DETAIL: "t0" is a nest table type +create table tableof_unsupported(id t2); +ERROR: type "t2" is not supported as column type +DETAIL: "t2" is a nest table type diff --git a/src/test/regress/sql/tableof_unsupported.sql b/src/test/regress/sql/tableof_unsupported.sql new file mode 100644 index 000000000..f574c6a66 --- /dev/null +++ b/src/test/regress/sql/tableof_unsupported.sql @@ -0,0 +1,12 @@ +------ Prepare ------ +-- create type of TYPTYPE_TABLEOF +create type r0 is (c1 int1, c2 int2); + +create type t0 is table of int4; +create type t1 is (c1 int1, c2 t0); +create type t2 is table of r0; + +-- create table +create table tableof_unsupported(id t0); +create table tableof_unsupported(id t1); +create table tableof_unsupported(id t2); \ No newline at end of file From 26befdedb6c368fe55faa7e9623ede8e05812dad Mon Sep 17 00:00:00 2001 From: luo_zihao5524 Date: Sat, 9 Sep 2023 19:12:57 +0800 Subject: [PATCH 223/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8DExecResult=E9=94=99?= =?UTF-8?q?=E8=AF=AF=E6=B8=85=E7=A9=BA=E5=86=85=E5=AD=98=E4=B8=8A=E4=B8=8B?= =?UTF-8?q?=E6=96=87=E7=9A=84bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/pl/plpgsql/src/gram.y | 12 +++++----- src/gausskernel/runtime/executor/execQual.cpp | 17 +++++++++++++ .../runtime/executor/nodeResult.cpp | 10 ++------ src/include/executor/executor.h | 1 + src/include/nodes/execnodes.h | 2 ++ .../regress/expected/exec_result_test.out | 24 +++++++++++++++++++ src/test/regress/parallel_schedule0A | 2 +- src/test/regress/sql/exec_result_test.sql | 19 +++++++++++++++ 8 files changed, 72 insertions(+), 15 deletions(-) create mode 100644 src/test/regress/expected/exec_result_test.out create mode 100644 src/test/regress/sql/exec_result_test.sql diff --git a/src/common/pl/plpgsql/src/gram.y b/src/common/pl/plpgsql/src/gram.y index 64d200c1f..7e6f69ed6 100755 --- a/src/common/pl/plpgsql/src/gram.y +++ b/src/common/pl/plpgsql/src/gram.y @@ -11290,8 +11290,8 @@ read_into_target(PLpgSQL_rec **rec, PLpgSQL_row **row, bool *strict, int firstto errmsg("Improper use of '.*'. The '.*' operator cannot be used with a row type variable."), parser_errposition(yylloc))); } - if (tok == T_DATUM || tok == T_VARRAY_VAR - || tok == T_TABLE_VAR || tok == T_PACKAGE_VARIABLE) { + if (!DB_IS_CMPT(PG_FORMAT) && (tok == T_DATUM || tok == T_VARRAY_VAR + || tok == T_TABLE_VAR || tok == T_PACKAGE_VARIABLE)) { const char* message = "syntax error, expected \",\""; InsertErrorMessage(message, plpgsql_yylloc); ereport(errstate, @@ -11314,8 +11314,8 @@ read_into_target(PLpgSQL_rec **rec, PLpgSQL_row **row, bool *strict, int firstto errmsg("record or row variable cannot be part of multiple-item INTO list"), parser_errposition(yylloc))); } - if (tok == T_DATUM || tok == T_VARRAY_VAR - || tok == T_TABLE_VAR || tok == T_PACKAGE_VARIABLE) { + if (!DB_IS_CMPT(PG_FORMAT) && (tok == T_DATUM || tok == T_VARRAY_VAR + || tok == T_TABLE_VAR || tok == T_PACKAGE_VARIABLE)) { const char* message = "syntax error, expected \",\""; InsertErrorMessage(message, plpgsql_yylloc); ereport(errstate, @@ -11742,8 +11742,8 @@ read_into_array_table_scalar_list(char *initial_name, } } - if (tok == T_DATUM || tok == T_VARRAY_VAR - || tok == T_TABLE_VAR || tok == T_PACKAGE_VARIABLE) { + if (!DB_IS_CMPT(PG_FORMAT) && (tok == T_DATUM || tok == T_VARRAY_VAR + || tok == T_TABLE_VAR || tok == T_PACKAGE_VARIABLE)) { const char* message = "syntax error, expected \",\""; InsertErrorMessage(message, plpgsql_yylloc); ereport(errstate, diff --git a/src/gausskernel/runtime/executor/execQual.cpp b/src/gausskernel/runtime/executor/execQual.cpp index 4965f4a03..a5e822161 100644 --- a/src/gausskernel/runtime/executor/execQual.cpp +++ b/src/gausskernel/runtime/executor/execQual.cpp @@ -1905,6 +1905,7 @@ static void init_fcache( fcache->funcResultSlot = NULL; fcache->setArgsValid = false; fcache->shutdown_reg = false; + fcache->setArgByVal = false; if(fcache->xprstate.is_flt_frame){ fcache->is_plpgsql_func_with_outparam = is_function_with_plpgsql_language_and_outparam(fcache->func.fn_oid); fcache->has_refcursor = func_has_refcursor_args(fcache->func.fn_oid, &fcache->fcinfo_data); @@ -1936,6 +1937,7 @@ extern void ShutdownFuncExpr(Datum arg) /* Clear any active set-argument state */ fcache->setArgsValid = false; + fcache->setArgByVal = false; /* execUtils will deregister the callback... */ fcache->shutdown_reg = false; @@ -2188,6 +2190,16 @@ void set_result_for_plpgsql_language_function_with_outparam(FuncExprState *fcach pfree(nulls); } +bool ExecSetArgIsByValue(FunctionCallInfo fcinfo) +{ + for (int i = 0; i < fcinfo->nargs; i++) { + if (!fcinfo->argnull[i] && !get_typbyval(fcinfo->argTypes[i])) { + return false; + } + } + return true; +} + /* * ExecMakeFunctionResult * @@ -2338,6 +2350,7 @@ static Datum ExecMakeFunctionResult(FuncExprState* fcache, ExprContext* econtext } else { hasSetArg = (argDone != ExprSingleResult); } + fcache->setArgByVal = ExecSetArgIsByValue(fcinfo); } else { /* Re-use callinfo from previous evaluation */ hasSetArg = fcache->setHasSetArg; @@ -2497,6 +2510,10 @@ static Datum ExecMakeFunctionResult(FuncExprState* fcache, ExprContext* econtext if (fcache->func.fn_retset && *isDone == ExprMultipleResult) { fcache->setHasSetArg = hasSetArg; fcache->setArgsValid = true; + /* arg not by value, memory can not be reset */ + if (!fcache->setArgByVal) { + econtext->hasSetResultStore = true; + } /* Register cleanup callback if we didn't already */ if (!fcache->shutdown_reg) { RegisterExprContextCallback(econtext, ShutdownFuncExpr, PointerGetDatum(fcache)); diff --git a/src/gausskernel/runtime/executor/nodeResult.cpp b/src/gausskernel/runtime/executor/nodeResult.cpp index 830b42d79..8ee6da135 100644 --- a/src/gausskernel/runtime/executor/nodeResult.cpp +++ b/src/gausskernel/runtime/executor/nodeResult.cpp @@ -91,13 +91,6 @@ static TupleTableSlot* ExecResult(PlanState* state) } } - /* - * Reset per-tuple memory context to free any expression evaluation - * storage allocated in the previous tuple cycle. Note this can't happen - * until we're done projecting out tuples from a scan tuple. - */ - ResetExprContext(econtext); - /* * Reset per-tuple memory context to free any expression evaluation * storage allocated in the previous tuple cycle. Note this can't happen @@ -124,7 +117,8 @@ static TupleTableSlot* ExecResult(PlanState* state) if (econtext->hasSetResultStore) { /* return values all store in ResultStore, could not free early one */ - + ResetExprContext(econtext); + econtext->hasSetResultStore = false; } /* diff --git a/src/include/executor/executor.h b/src/include/executor/executor.h index 14ee0ad6a..49358b835 100755 --- a/src/include/executor/executor.h +++ b/src/include/executor/executor.h @@ -246,6 +246,7 @@ extern void EvalPlanQualFetchRowMarksUHeap(EPQState* epqstate); extern TupleTableSlot* EvalPlanQualNext(EPQState* epqstate); extern void EvalPlanQualBegin(EPQState* epqstate, EState* parentestate, bool isUHeap = false); extern void EvalPlanQualEnd(EPQState* epqstate); +extern bool ExecSetArgIsByValue(FunctionCallInfoData* fcinfo); /* * functions in execProcnode.c diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index fbc391a2f..d4fa660ae 100755 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -964,6 +964,8 @@ typedef struct FuncExprState { */ bool setArgsValid; + bool setArgByVal; /* all args are by val? */ + bool setHasSetArg; bool is_plpgsql_func_with_outparam; diff --git a/src/test/regress/expected/exec_result_test.out b/src/test/regress/expected/exec_result_test.out new file mode 100644 index 000000000..35231f187 --- /dev/null +++ b/src/test/regress/expected/exec_result_test.out @@ -0,0 +1,24 @@ +drop schema if exists orafce cascade; +NOTICE: schema "orafce" does not exist, skipping +create schema orafce; +CREATE OR REPLACE FUNCTION orafce.regexp_substr(text, text, integer, integer, text, int) +RETURNS text +AS $$ +DECLARE + v_substr text; + v_pattern text := $2; + modifiers text := $5; + v_subexpr integer := $6; + has_group integer := 2; +BEGIN + v_substr := (SELECT (regexp_matches(substr($1, $3), v_pattern, modifiers))[v_subexpr] OFFSET $4 - 1 LIMIT 1); + RETURN v_substr; +END; +$$ +LANGUAGE plpgsql; +SELECT orafce.REGEXP_SUBSTR('1234567890 1234557890', '(123)(4(5[56])(78))', 1, 2, 'i', 3); + regexp_substr +--------------- + +(1 row) + diff --git a/src/test/regress/parallel_schedule0A b/src/test/regress/parallel_schedule0A index 1be0c5363..d5b012998 100644 --- a/src/test/regress/parallel_schedule0A +++ b/src/test/regress/parallel_schedule0A @@ -29,7 +29,7 @@ test: replace_func_with_two_args trunc_func_for_date nlssort_pinyin updatable_vi # test multiple statistics test: functional_dependency -test: pg_proc_test test_row_type_in_proc +test: pg_proc_test test_row_type_in_proc exec_result_test # test fdw # NOTICE: In the "fdw_prepare", we copy the fdw test to be used from contrib into regress sql set. diff --git a/src/test/regress/sql/exec_result_test.sql b/src/test/regress/sql/exec_result_test.sql new file mode 100644 index 000000000..ee522bc37 --- /dev/null +++ b/src/test/regress/sql/exec_result_test.sql @@ -0,0 +1,19 @@ +drop schema if exists orafce cascade; +create schema orafce; +CREATE OR REPLACE FUNCTION orafce.regexp_substr(text, text, integer, integer, text, int) +RETURNS text +AS $$ +DECLARE + v_substr text; + v_pattern text := $2; + modifiers text := $5; + v_subexpr integer := $6; + has_group integer := 2; +BEGIN + v_substr := (SELECT (regexp_matches(substr($1, $3), v_pattern, modifiers))[v_subexpr] OFFSET $4 - 1 LIMIT 1); + RETURN v_substr; +END; +$$ +LANGUAGE plpgsql; + +SELECT orafce.REGEXP_SUBSTR('1234567890 1234557890', '(123)(4(5[56])(78))', 1, 2, 'i', 3); From 147d9d2c1c3ce67d00ab84dcb50d4d528f6f3114 Mon Sep 17 00:00:00 2001 From: "arcoalien@qq.com" Date: Tue, 12 Sep 2023 19:11:09 +0800 Subject: [PATCH 224/304] =?UTF-8?q?1.=E8=A7=A3=E5=86=B3=E4=B8=BB=E6=9C=BA?= =?UTF-8?q?=E6=96=B0=E5=8A=A0=E6=AE=B5=E9=A1=B5=E5=BC=8F=E6=96=87=E4=BB=B6?= =?UTF-8?q?=EF=BC=8C=E5=A4=87=E6=9C=BA=E6=9B=B4=E6=96=B0=E6=97=B6=E6=B2=A1?= =?UTF-8?q?=E6=9C=89=E5=88=9D=E5=A7=8B=E5=8C=96=E7=9A=84=E9=97=AE=E9=A2=98?= =?UTF-8?q?=202.=E8=A7=A3=E5=86=B3debug=E6=A8=A1=E5=BC=8F=E4=B8=8Bcm=5Fage?= =?UTF-8?q?nt=E7=94=B1=E4=BA=8Eu=5Fsess=E4=B8=8A=E7=9A=84globalxmin?= =?UTF-8?q?=E6=98=AF=E6=97=A0=E6=95=88=E5=80=BC=EF=BC=8C=E5=AF=BC=E8=87=B4?= =?UTF-8?q?=E5=BC=82=E5=B8=B8=E7=9A=84=E9=97=AE=E9=A2=98=203.=E8=A7=A3?= =?UTF-8?q?=E5=86=B3=E4=B8=BB=E6=9C=BAdrop=20db/drop=20tablespace=EF=BC=8C?= =?UTF-8?q?=E5=B9=BF=E6=92=AD=E5=88=B0=E5=A4=87=E6=9C=BA=E5=90=8E=EF=BC=8C?= =?UTF-8?q?=E5=A4=87=E6=9C=BAmes=E5=A6=82=E6=9E=9C=E5=BC=82=E5=B8=B8?= =?UTF-8?q?=E9=80=80=E5=87=BA=EF=BC=8C=E4=B8=8D=E9=87=8A=E6=94=BE=E9=94=81?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ddes/adapter/ss_transaction.cpp | 4 ++++ .../storage/smgr/segment/data_file.cpp | 5 ++++ .../storage/smgr/segment/space.cpp | 23 +++++++++++-------- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/gausskernel/ddes/adapter/ss_transaction.cpp b/src/gausskernel/ddes/adapter/ss_transaction.cpp index a25033bb7..642ebbe0b 100644 --- a/src/gausskernel/ddes/adapter/ss_transaction.cpp +++ b/src/gausskernel/ddes/adapter/ss_transaction.cpp @@ -102,6 +102,10 @@ Snapshot SSGetSnapshotData(Snapshot snapshot) /* For cm agent, it only query the system status using the parameter in memory. So don't need MVCC */ if (u_sess->libpq_cxt.IsConnFromCmAgent) { snapshot = SnapshotNow; + if (!TransactionIdIsNormal(u_sess->utils_cxt.RecentGlobalXmin)) { + u_sess->utils_cxt.RecentGlobalXmin = FirstNormalTransactionId; + } + u_sess->utils_cxt.RecentGlobalDataXmin = u_sess->utils_cxt.RecentGlobalXmin; return snapshot; } diff --git a/src/gausskernel/storage/smgr/segment/data_file.cpp b/src/gausskernel/storage/smgr/segment/data_file.cpp index 523c83f3b..6ea800893 100644 --- a/src/gausskernel/storage/smgr/segment/data_file.cpp +++ b/src/gausskernel/storage/smgr/segment/data_file.cpp @@ -155,6 +155,11 @@ bool df_ss_update_segfile_size(SegLogicFile *sf, BlockNumber target_block) } uint32 flags = O_RDWR | PG_BINARY; + /* need palloc segfiles if file_num is 0 */ + if (sf->vector_capacity == 0) { + df_extend_file_vector(sf); + } + if (sf->file_num == 0) { char *filename = slice_filename(sf->filename, 0); int fd = dv_open_file(filename, flags, (int)SEGMENT_FILE_MODE); diff --git a/src/gausskernel/storage/smgr/segment/space.cpp b/src/gausskernel/storage/smgr/segment/space.cpp index 122ff7884..bb8fcc0c5 100644 --- a/src/gausskernel/storage/smgr/segment/space.cpp +++ b/src/gausskernel/storage/smgr/segment/space.cpp @@ -413,19 +413,22 @@ static void SSClose_seg_files(SegSpace *spc) void SSDrop_seg_space(Oid spcNode, Oid dbNode) { - SegSpace *spc = spc_init_space_node(spcNode, dbNode); - AutoMutexLock spc_lock(&spc->lock); + SegSpace *entry = NULL; + AutoMutexLock spc_lock(&segspace_lock); + SegSpcTag tag = {.spcNode = spcNode, .dbNode = dbNode}; + SegmentCheck(t_thrd.storage_cxt.SegSpcCache != NULL); spc_lock.lock(); + entry = (SegSpace *)hash_search(t_thrd.storage_cxt.SegSpcCache, (void *)&tag, HASH_FIND, NULL); + spc_lock.unLock(); - SpaceDataFileStatus dataStatus = spc_status(spc); - if (dataStatus == SpaceDataFileStatus::EMPTY) { - spc_lock.unLock(); - return; + if (entry != NULL) { + if(entry->status == OPENED) { + SSClose_seg_files(entry); + SegDropSpaceMetaBuffers(spcNode, dbNode); + } + spc_lock.lock(); + (void)hash_search(t_thrd.storage_cxt.SegSpcCache, (void *)&tag, HASH_REMOVE, NULL); } - - SegDropSpaceMetaBuffers(spcNode, dbNode); - SSClose_seg_files(spc); - spc_lock.unLock(); return; } From 70a5ecaf62447170cfda0cf810845612836b5b97 Mon Sep 17 00:00:00 2001 From: "arcoalien@qq.com" Date: Tue, 12 Sep 2023 19:19:15 +0800 Subject: [PATCH 225/304] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E5=86=99=E8=BD=AC?= =?UTF-8?q?=E5=8F=91=EF=BC=8C=E5=A4=87=E6=9C=BA=E6=89=A7=E8=A1=8C=E4=BA=8B?= =?UTF-8?q?=E5=8A=A1=E5=AE=8C=E6=88=90=E5=90=8E=E6=89=A7=E8=A1=8Cddl?= =?UTF-8?q?=E8=AF=AD=E5=8F=A5=E5=A4=B1=E8=B4=A5=E6=8A=A5=E9=94=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/storage/replication/libpqsw.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/gausskernel/storage/replication/libpqsw.cpp b/src/gausskernel/storage/replication/libpqsw.cpp index b08333ba9..e03246af2 100644 --- a/src/gausskernel/storage/replication/libpqsw.cpp +++ b/src/gausskernel/storage/replication/libpqsw.cpp @@ -1094,6 +1094,10 @@ bool libpqsw_process_query_message(const char* commandTag, List* query_list, con libpqsw_set_set_command(false); libpqsw_set_redirect(false); + if (libpqsw_end_command(commandTag)) { + libpqsw_set_transaction(false); + } + if (get_redirect_manager()->ss_standby_state & SS_STANDBY_REQ_SELECT) { need_redirect = false; } From df409ab73f965adaf9c4f527d4b3ba81f95083bc Mon Sep 17 00:00:00 2001 From: xuxinxin Date: Sat, 2 Sep 2023 14:40:17 +0800 Subject: [PATCH 226/304] =?UTF-8?q?=E5=90=8C=E6=AD=A5dss=5Fcontrl?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/ddes/script/dss_contrl.sh | 294 ++++++++++++---------- 1 file changed, 158 insertions(+), 136 deletions(-) diff --git a/src/gausskernel/ddes/script/dss_contrl.sh b/src/gausskernel/ddes/script/dss_contrl.sh index a4b4deea6..c05a741ce 100644 --- a/src/gausskernel/ddes/script/dss_contrl.sh +++ b/src/gausskernel/ddes/script/dss_contrl.sh @@ -5,6 +5,7 @@ export LD_LIBRARY_PATH=${GAUSSHOME}/lib:${GAUSSHOME}/add-ons:$LD_LIBRARY_PATH curr_path=`dirname $(readlink -f $0)` curr_filename=`basename $(readlink -f $0)` os_user=`whoami` + file_user=`ls -l ${curr_path}"/${curr_filename}" | awk '{print $3}'` if [ ${file_user} != ${os_user} ]; then @@ -12,7 +13,10 @@ if [ ${file_user} != ${os_user} ]; then exit 1 fi -GSDB_BIN=${GAUSSHOME}/bin/gaussdb +GSDB_BIN=gaussdb +GSDB_BIN_FULL=${GAUSSHOME}/bin/gaussdb +DSS_BIN=dssserver +DSS_BIN_FULL=${GAUSSHOME}/bin/dssserver BIN_PATH=${GAUSSHOME}/bin SCRIPT_NAME=$0 @@ -45,7 +49,7 @@ fi log() { time=`date "+%Y-%m-%d %H:%M:%S"` - echo "$time $1" + echo "[$time][DSS]$1" >> ${startdss_log} 2>&1 } assert_empty() @@ -57,17 +61,43 @@ assert_nonempty() { if [[ -z ${2} ]] then - log "The ${1} parameter is empty." + log "[SCRIPT]The ${1} parameter is empty." exit 1 fi } program_pid() { - pid=`ps -f f -u \`whoami\` | grep ${1} | grep ${2} | grep -v grep | grep -v ${SCRIPT_NAME} | awk '{print $2}'` + pid=`ps -f f -u \`whoami\` | grep -w ${1} | grep ${2} | grep -v grep | grep -v ${SCRIPT_NAME} | awk '{print $2}' | tail -1` echo ${pid} } +program_pid2() +{ + pid=`ps -f f -u \`whoami\` | grep -w ${1} | grep -v grep | grep -v ${SCRIPT_NAME} | awk '{print $2}'` + echo ${pid} +} + +program_status() +{ + pid=`program_pid $1 $2` + if [[ -z ${pid} ]]; then + echo "" + return + fi + + pstatus_file="/proc/"${pid}"/status" + cat ${pstatus_file} | while read line + do + if [[ "${line}" =~ ^State.* ]]; then + echo ${line} | awk -F ":" '{print $2}' | awk -F " " '{print $1}' + return + fi + done + + echo "" +} + kill_program() { assert_nonempty 1 ${1} @@ -75,39 +105,30 @@ kill_program() pid=`program_pid $1 $2` if [[ -z ${pid} ]] then - log "${1} is already dead." + log "[KILL]${1} is already dead." return fi kill -9 ${pid} - sleep 3 - ps -f -p "${pid}" | grep ${1} - if [ $? = 0 ] + if [ $? -ne 0 ] then - log "ERROR! ${1} with pid:${pid} is not killed..." - exit 0 + log "[KILL]ERROR! ${1} with pid:${pid} is not killed..." + exit 1 fi -} - -check_dss_start() -{ - started=0 - for (( i=1; i<30; i++ )) + for ((i=0; i < 30; i++)) do - pid=`program_pid dssserver ${1}` - if [[ ! -z ${pid} ]] + ps -f -p "${pid}" | grep ${1} + if [ $? -eq 0 ] then - started=1 - break + sleep 0.1 + else + log "[KILL]SUCCESS!" + return fi - sleep 1 done - if [[ ${started} -eq 0 ]] - then - log "ERROR! start dssserver in dir ${1} failed" - exit 1 - fi + log "[KILL]ERROR! ${1} with pid:${pid} is not killed..." + exit 1 } function clear_script_log @@ -119,13 +140,13 @@ function clear_script_log if [ -L ${_log_dir} ]; then typeset log_num=`find -L "${_log_dir}" -maxdepth 1 -type f -name "${_log_name}*" | wc -l` if [ ${log_num} -ge ${_max_log_backup} ];then - find -L "${_log_dir}" -maxdepth 1 -type f -name "${_log_name}*" | xargs ls -t {} 2>/dev/null | tail -n $(expr ${log_num} - ${_max_log_backup}) | xargs -i rm -f {} + find -L "${_log_dir}" -maxdepth 1 -type f -name "${_log_name}*" | xargs ls -t {} 2>/dev/null | tail -n $(expr ${log_num} - ${_max_log_backup}) | xargs -i rm -f {} fi else - typeset log_num=$(find "${_log_dir}" -maxdepth 1 -type f -name "${_log_name}*" | wc -l) + typeset log_num=`find -L "${_log_dir}" -maxdepth 1 -type f -name "${_log_name}*" | wc -l` if [ ${log_num} -ge ${_max_log_backup} ];then - find "${_log_dir}" -maxdepth 1 -type f -name "${_log_name}*" | xargs ls -t {} 2>/dev/null | tail -n $(expr ${log_num} - ${_max_log_backup}) | xargs -i rm -f {} - fi + find "${_log_dir}" -maxdepth 1 -type f -name "${_log_name}*" | xargs ls -t {} 2>/dev/null | tail -n $(expr ${log_num} - ${_max_log_backup}) | xargs -i rm -f {} + fi fi } @@ -140,7 +161,7 @@ check_log_file() log_file_size=$(ls -l ${log_file} |awk '{print $5}') if [ -f ${log_file} ];then if [ ${log_file_size} -ge ${MAX_LOG_SIZE} ];then - mv -f ${log_file} "${log_path}/${operation}-`date +%Y-%m-%d_%H%M%S`.log" 2>/dev/null + mv -f ${log_file} "${log_path}/${operation} - `date +%Y-%m-%d_%H%M%S`.log" 2>/dev/null clear_script_log "${log_path}" "${operation}-" $MAX_LOG_BACKUP fi fi @@ -152,6 +173,7 @@ touch_logfile() if [ ! -f $log_file ] then touch $log_file + chmod 600 $log_file fi } @@ -165,32 +187,48 @@ INSTANCE_ID=${2} export DSS_HOME=${3} GSDB_HOME=${4} CONN_PATH=UDS:${DSS_HOME}/.dss_unix_d_socket +startdss_log=${DSS_HOME}/startdss.log function check_dss_config() { - log "Checking dss_inst.ini before start dss..." + log "[START]Checking dss_inst.ini before start dss..." if [[ ! -e ${DSS_HOME}/cfg/dss_inst.ini ]] then - log "${DSS_HOME}/cfg/dss_inst.ini must exist" + log "[START]${DSS_HOME}/cfg/dss_inst.ini must exist" exit 1 fi - log "Checking dss_vg_conf.ini before start dss..." + log "[START]Checking dss_vg_conf.ini before start dss..." if [[ ! -e ${DSS_HOME}/cfg/dss_vg_conf.ini ]] then - log "${DSS_HOME}/cfg/dss_vg_conf.ini must exist" + log "[START]${DSS_HOME}/cfg/dss_vg_conf.ini must exist" exit 1 fi - LSNR_PATH=`awk '/LSNR_PATH/{print}' ${DSS_HOME}/cfg/dss_inst.ini | awk -F= '{print $2}' | xargs` + LSNR_PATH=`cat ${DSS_HOME}/cfg/dss_inst.ini | sed s/[[:space:]]//g |grep -Eo "^LSNR_PATH=.*" | awk -F '=' '{print $2}'` if [[ -z ${LSNR_PATH} ]] then - log "can't find lsnr path. Aborting." + log "[START]can't find lsnr path. Aborting." exit 1 fi CONN_PATH=UDS:${LSNR_PATH}/.dss_unix_d_socket } +get_startdss_log() +{ + LOG_HOME=`cat ${DSS_HOME}/cfg/dss_inst.ini | sed s/[[:space:]]//g |grep -Eo "^LOG_HOME=.*" | awk -F '=' '{print $2}'` + if [[ ! -z ${LOG_HOME} ]] + then + startdss_log=${LOG_HOME}/startdss.log + fi + + if [[ -z ${DSS_HOME} ]] + then + startdss_log=/dev/null + else + touch_logfile $startdss_log + fi +} function ScandCheck() { groups=`groups` @@ -209,49 +247,52 @@ function ScandCheck() done } +kill_dss_and_perctrl() +{ + pid=$(program_pid dssserver ${DSS_HOME}) + if [[ -z ${pid} ]] + then + log "[${1}]dssserver not exist." + fi + kill_program dssserver ${DSS_HOME} + log "[${1}]Success to kill dssserver." + + pid=$(program_pid2 perctrl) + for perctrl_pid in ${pid} + do + if [[ -z ${perctrl_pid} ]] + then + log "[${1}]perctrl not exist." + fi + kill_program ${perctrl_pid} perctrl + log "[${1}]kill perctrl ${perctrl_pid} success." + done +} + # 1st step: if database exists, kill it # 2nd step: if dssserver no exists, start it function Start() { check_dss_config - - startdss_log=${DSS_HOME}/startdss.log - db_start_log=${GSDB_HOME}/DBstart.log check_log_file ${DSS_HOME} $startdss_log startdss - check_log_file ${GSDB_HOME} $db_start_log DBstart - if [[ -z "${DSS_HOME}" ]] - then - startdss_log=/dev/null - else - touch_logfile $startdss_log - chmod 600 $startdss_log - fi - - if [[ -z "${GSDB_HOME}" ]] - then - db_start_log=/dev/null - else - touch_logfile $db_start_log - chmod 600 $db_start_log - fi - - pid=`program_pid dssserver ${DSS_HOME}` + pid=`program_pid ${DSS_BIN_FULL} ${DSS_HOME}` if [[ ! -z ${pid} ]] then - log "dssserver already started in dir ${DSS_HOME}..." + log "[START]dssserver already started in dir ${DSS_HOME}..." else - log "Starting dssserver..." - pid=`program_pid ${GSDB_BIN} ${GSDB_HOME}` + log "[START]Starting dssserver..." + pid=`program_pid ${GSDB_BIN_FULL} ${GSDB_HOME}` if [[ ! -z ${pid} ]] then - kill_program ${GSDB_BIN} ${GSDB_HOME} + log "[START]kill ${GSDB_BIN} before start dssserver" + kill_program ${GSDB_BIN_FULL} ${GSDB_HOME} else - log "${GSDB_BIN} is offline in dir ${GSDB_HOME}..." + log "[START]${GSDB_BIN} is offline in dir ${GSDB_HOME}..." fi + log "[START]dssserver" ScandCheck - nohup dssserver -D ${DSS_HOME} >> ${startdss_log} 2>&1 & - check_dss_start ${DSS_HOME} - log "start dss in ${DSS_HOME} success." + nohup ${DSS_BIN_FULL} -D ${DSS_HOME} >> ${startdss_log} 2>&1 & + log "[START]start dssserver in ${DSS_HOME} is starting." fi } @@ -262,94 +303,71 @@ function Start() # 5rd step: if fail to stop dssserver in 2nd step, then kill dssserver function Stop() { - log "stop ${GSDB_BIN}..." + check_log_file ${DSS_HOME} $startdss_log startdss + + log "[STOP]stop ${GSDB_BIN}..." db_flag_file=instance_manual_start_$(expr $INSTANCE_ID + 6001) - echo "db_flag_file=$db_flag_file" + log "[STOP]db_flag_file=$db_flag_file" if [[ -f $GAUSSHOME/bin/$db_flag_file ]]; then - log "$GAUSSHOME/bin/$db_flag_file is exist" + log "[STOP]$GAUSSHOME/bin/$db_flag_file is exist" else touch $GAUSSHOME/bin/$db_flag_file fi - pid=$(program_pid ${GSDB_BIN} ${GSDB_HOME}) - if [[ -z ${pid} ]] + pid=$(program_pid ${GSDB_BIN_FULL} ${GSDB_HOME}) + if [[ ! -z ${pid} ]] then - log "stop dssserver if running..." - nohup dsscmd stopdss -U ${CONN_PATH} >> /dev/null 2>&1 - sleep 2 - - pid=`program_pid dssserver ${DSS_HOME}` - if [[ -z ${pid} ]] - then - log "dssserver stopped in dir ${DSS_HOME}..." - exit 0 - fi - log "Killing dssserver if running..." - kill_program dssserver ${DSS_HOME} - else - log "stop ${GSDB_BIN}..." - ${BIN_PATH}/gs_ctl stop -D ${GSDB_HOME} - sleep 5 - - pid=`program_pid ${GSDB_BIN} ${GSDB_HOME}` - if [[ -z ${pid} ]] - then - log "${GSDB_BIN} stopped in dir ${GSDB_HOME}..." - else - log "Killing ${GSDB_BIN} if running..." - kill_program ${GSDB_BIN} ${GSDB_HOME} - fi - - log "stop dssserver if running..." - nohup dsscmd stopdss -U ${CONN_PATH} >> /dev/null 2>&1 - sleep 2 - pid=`program_pid dssserver ${DSS_HOME}` - if [[ -z ${pid} ]] - then - log "dssserver stopped in dir ${DSS_HOME}..." - exit 0 - fi - log "Killing dssserver if running..." - kill_program dssserver ${DSS_HOME} + log "[STOP] kill ${GSDB_BIN} before stop dssserver" + kill_program ${GSDB_BIN_FULL} ${GSDB_HOME} fi + + kill_dss_and_perctrl "STOP" } # 1st step: check dssserver if exists + + + function Check() { - pid=$(program_pid dssserver ${DSS_HOME}) - if [[ -z ${pid} ]] + dss_status=$(program_status dssserver ${DSS_HOME}) + if [[ -z ${dss_status} ]] then - log "check dssserver in ${DSS_HOME} fail." + log "[CHECK]dssserver is offline." exit 1 fi + if [[ "${dss_status}" == "D" || "${dss_status}" == "T" || "${dss_status}" == "Z" ]] + then + log "[CHECK]dssserver is dead." + exit 3 + fi - log "check dss in ${DSS_HOME} success." + pid=$(program_pid2 perctrl) + for perctrl_pid in ${pid} + do + perctrl_status=$(program_status ${perctrl_pid} perctrl) + if [[ "${perctrl_status}" == "D" || "${perctrl_status}" == "T" || "${perctrl_status}" == "Z" ]] + then + log "[CHECK]perctrl is dead." + exit 3 + fi + done } - # 1st step: kill database # 2nd step: stop dssserver by using dsscmd # 3rd step: if fail to stop dssserver in 2nd step, then kill dssserver function Clean() { - log "stop ${GSDB_BIN}..." - kill_program ${GSDB_BIN} ${GSDB_HOME} - sleep 3 - - log "stop dssserver if running..." - nohup dsscmd stopdss -U ${CONN_PATH} >> /dev/null 2>&1 - sleep 2 - - pid=`program_pid dssserver ${DSS_HOME}` - if [[ -z ${pid} ]] + check_log_file ${DSS_HOME} $startdss_log startdss + pid=$(program_pid ${GSDB_BIN_FULL} ${GSDB_HOME}) + if [[ ! -z ${pid} ]] then - log "dssserver stopped in dir ${DSS_HOME}..." - exit 0 + log "[CLEAN]kill ${GSDB_BIN} before kill dssserver" + kill_program ${GSDB_BIN_FULL} ${GSDB_HOME} fi - log "Killing dssserver if running..." - kill_program dssserver ${DSS_HOME} + kill_dss_and_perctrl "CLEAN" dsscmd clean_vglock -D ${DSS_HOME} >> /dev/null 2>&1 } @@ -359,16 +377,16 @@ function Reg() LOCAL_INSTANCE_ID=`awk '/INST_ID/{print}' ${DSS_HOME}/cfg/dss_inst.ini | awk -F= '{print $2}' | xargs` if [[ -z ${LOCAL_INSTANCE_ID} ]] then - log "can't find inst id. Aborting." + log "[REG]can't find inst id. Aborting." exit 1 fi dsscmd reghl -D ${DSS_HOME} >> /dev/null 2>&1 if [[ $? != 0 ]] then - log "dsscmd reghl -D ${DSS_HOME} fail." + log "[REG]dsscmd reghl -D ${DSS_HOME} fail." exit 1 fi - log "register success." + log "[REG]register success." } function Unreg() @@ -376,7 +394,7 @@ function Unreg() LOCAL_INSTANCE_ID=`awk '/INST_ID/{print}' ${DSS_HOME}/cfg/dss_inst.ini | awk -F= '{print $2}' | xargs` if [[ -z ${LOCAL_INSTANCE_ID} ]] then - log "can't find inst id. Aborting." + log "[UNREG]can't find inst id. Aborting." exit 1 fi if [[ ${LOCAL_INSTANCE_ID} == ${INSTANCE_ID} ]] @@ -386,7 +404,7 @@ function Unreg() pid=$(program_pid dssserver ${DSS_HOME}) if [[ -z ${pid} ]] then - log "dssserver is not running." + log "[UNREG]dssserver is not running." exit 1 fi dsscmd kickh -i ${INSTANCE_ID} -D ${DSS_HOME} >> /dev/null 2>&1 @@ -394,10 +412,10 @@ function Unreg() if [[ $? != 0 ]] then - log "dsscmd kickh -i ${INSTANCE_ID} -D ${DSS_HOME} fail, or dsscmd unreghl -D ${DSS_HOME} fail." + log "[UNREG]dsscmd kickh -i ${INSTANCE_ID} -D ${DSS_HOME} fail, or dsscmd unreghl -D ${DSS_HOME} fail." exit 1 fi - log "unregister ${INSTANCE_ID} success." + log "[UNREG]unregister ${INSTANCE_ID} success." } function Isreg() @@ -406,9 +424,13 @@ function Isreg() result=$? if [[ ${result} == 255 ]] then - log "dsscmd inq_reg -i ${INSTANCE_ID} -D ${DSS_HOME} fail." + log "[ISREG]dsscmd inq_reg -i ${INSTANCE_ID} -D ${DSS_HOME} fail." exit -1 fi + if [[ ${result} != 2 ]] + then + log "[ISREG]result: ${result}" + fi exit ${result} } @@ -436,7 +458,7 @@ function Main() Isreg exit 0 else - echo "Please confirm the input parameters." + echo "[SCRIPT]Please confirm the input parameters." exit 1 fi } From 4fd7d264f12494bdbce0fc73f3e18479ee000171 Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Fri, 8 Sep 2023 16:35:16 +0800 Subject: [PATCH 227/304] =?UTF-8?q?dss=209.12=E6=8E=A8=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/utils/error/elog.cpp | 1 - src/gausskernel/ddes/ddes_commit_id | 2 +- src/gausskernel/storage/dss/fio_dss.cpp | 16 +++++++++-- src/gausskernel/storage/file/fio_device.cpp | 2 +- src/include/storage/dss/dss_api_def.h | 30 +++++++++++++++------ src/include/storage/smgr/smgr.h | 2 +- 6 files changed, 39 insertions(+), 14 deletions(-) diff --git a/src/common/backend/utils/error/elog.cpp b/src/common/backend/utils/error/elog.cpp index 22454bfc2..3fb6f5600 100644 --- a/src/common/backend/utils/error/elog.cpp +++ b/src/common/backend/utils/error/elog.cpp @@ -897,7 +897,6 @@ int errcode_for_file_access(void) /* File not found */ case ENOENT: /* No such file or directory */ case ERR_DSS_FILE_NOT_EXIST: /* No such file in dss */ - case ERR_DSS_DIR_NOT_EXIST: /* No such directory in dss */ edata->sqlerrcode = ERRCODE_UNDEFINED_FILE; break; diff --git a/src/gausskernel/ddes/ddes_commit_id b/src/gausskernel/ddes/ddes_commit_id index 811136323..6eb39ddee 100644 --- a/src/gausskernel/ddes/ddes_commit_id +++ b/src/gausskernel/ddes/ddes_commit_id @@ -1,3 +1,3 @@ dms_commit_id=d7095d9b169a766461b00aa4bc0e19b4fc8ea657 -dss_commit_id=87adc9bafa80354bb138a6bf9f8ff63077f3aebf +dss_commit_id=0a759717486f91cc91b19ca6b0cf0d2e48f08fcb cbb_commit_id=7a7e77f3dec94b6b958bea12c97b92d21dfa0bb5 \ No newline at end of file diff --git a/src/gausskernel/storage/dss/fio_dss.cpp b/src/gausskernel/storage/dss/fio_dss.cpp index 3feac456c..60ad17bba 100644 --- a/src/gausskernel/storage/dss/fio_dss.cpp +++ b/src/gausskernel/storage/dss/fio_dss.cpp @@ -80,20 +80,32 @@ bool is_dss_fd(int handle) return false; } +int parse_errcode_from_errormsg(const char* errormsg) { + const char *errcode_str = strstr(errormsg, "errcode:"); + if (errcode_str) { + errcode_str += strlen("errcode:"); + return atoi(errcode_str); + } + return ERR_DSS_PROCESS_REMOTE; +} + void dss_set_errno(int *errcode) { int errorcode = 0; const char *errormsg = NULL; g_dss_device_op.dss_get_error(&errorcode, &errormsg); - errno = errorcode; + if (errorcode == ERR_DSS_PROCESS_REMOTE) { + errno = parse_errcode_from_errormsg(errormsg); + } else { + errno = errorcode; + } if (errcode != NULL) { *errcode = errorcode; } } - int dss_access_file(const char *file_name, int mode) { struct stat statbuf = {0}; diff --git a/src/gausskernel/storage/file/fio_device.cpp b/src/gausskernel/storage/file/fio_device.cpp index 021351115..8fff4e5b7 100644 --- a/src/gausskernel/storage/file/fio_device.cpp +++ b/src/gausskernel/storage/file/fio_device.cpp @@ -50,5 +50,5 @@ bool is_file_exist(int err) bool is_file_delete(int err) { - return (err == ENOENT || err == ERR_DSS_DIR_NOT_EXIST || err == ERR_DSS_FILE_NOT_EXIST); + return (err == ENOENT || err == ERR_DSS_FILE_NOT_EXIST); } \ No newline at end of file diff --git a/src/include/storage/dss/dss_api_def.h b/src/include/storage/dss/dss_api_def.h index 7e9b62920..7c97a9a48 100644 --- a/src/include/storage/dss/dss_api_def.h +++ b/src/include/storage/dss/dss_api_def.h @@ -93,8 +93,7 @@ typedef void (*dss_log_output)(dss_log_id_t log_type, dss_log_level_t log_level, * 2.ERR_DSS_SUBMODEL_ACTION_DETAIL, _DETAIL is optional which indicates the error cause. */ #define ERR_DSS_FLOOR 2000 - -// vg error [2000, 2060) +// vg error [2000, 2050) #define ERR_DSS_VG_CREATE 2000 #define ERR_DSS_VG_LOCK 2010 #define ERR_DSS_VG_REMOVE 2020 @@ -102,7 +101,8 @@ typedef void (*dss_log_output)(dss_log_id_t log_type, dss_log_level_t log_level, #define ERR_DSS_VG_CHECK_NOT_INIT 2031 #define ERR_DSS_VG_NOT_EXIST 2040 -// volumn error [2060, 2130) +// volumn error [2050, 2130) +#define ERR_DSS_VOLUME_SYSTEM_IO 2050 #define ERR_DSS_VOLUME_OPEN 2060 #define ERR_DSS_VOLUME_READ 2070 #define ERR_DSS_VOLUME_WRITE 2080 @@ -113,24 +113,31 @@ typedef void (*dss_log_output)(dss_log_id_t log_type, dss_log_level_t log_level, #define ERR_DSS_VOLUME_REMOVE_NOEXIST 2111 #define ERR_DSS_VOLUME_REMOVE_NONEMPTY 2112 #define ERR_DSS_VOLUME_REMOVE_SUPER_BLOCK 2113 - + // file error [2130, 2230) #define ERR_DSS_FILE_SEEK 2130 #define ERR_DSS_FILE_REMOVE 2140 #define ERR_DSS_FILE_REMOVE_OPENING 2141 #define ERR_DSS_FILE_RENAME 2150 +#define ERR_DSS_FILE_RENAME_DIFF_VG 2151 +#define ERR_DSS_FILE_RENAME_EXIST 2152 +#define ERR_DSS_FILE_RENAME_OPENING_REMOTE 2153 +#define ERR_DSS_FILE_CLOSE 2160 +#define ERR_DSS_FILE_CREATE 2170 +#define ERR_DSS_FILE_RDWR 2180 +#define ERR_DSS_FILE_RDWR_INSUFF_PER 2181 #define ERR_DSS_FILE_NOT_EXIST 2190 #define ERR_DSS_FILE_OPENING_REMOTE 2191 #define ERR_DSS_FILE_TYPE_MISMATCH 2192 #define ERR_DSS_FILE_PATH_ILL 2193 +#define ERR_DSS_FILE_INVALID_SIZE 2194 +#define ERR_DSS_FILE_INVALID_WRITTEN_SIZE 2195 // dir error [2230, 2280) #define ERR_DSS_DIR_REMOVE 2230 #define ERR_DSS_DIR_REMOVE_NOT_EMPTY 2231 #define ERR_DSS_DIR_CREATE 2240 #define ERR_DSS_DIR_CREATE_DUPLICATED 2241 -#define ERR_DSS_DIR_NOT_EXIST 2270 - // link error [2280, 2300) #define ERR_DSS_LINK_READ 2280 #define ERR_DSS_LINK_READ_NOT_LINK 2281 @@ -153,7 +160,9 @@ typedef void (*dss_log_output)(dss_log_id_t log_type, dss_log_level_t log_level, #define ERR_DSS_SKLIST_EXIST 2363 #define ERR_DSS_SHM_CREATE 2370 #define ERR_DSS_SHM_CHECK 2371 +#define ERR_DSS_SHM_LOCK 2372 #define ERR_DSS_GA_INIT 2380 +#define ERR_DSS_GA_GET_ADDR 2381 #define ERR_DSS_SESSION_INVALID_ID 2390 #define ERR_DSS_SESSION_CREATE 2391 @@ -171,8 +180,13 @@ typedef void (*dss_log_output)(dss_log_id_t log_type, dss_log_level_t log_level, #define ERR_DSS_TCP_TIMEOUT_REMAIN 2410 #define ERR_DSS_UDS_INVALID_URL 2411 #define ERR_DSS_RECV_MSG_FAILED 2412 -#define ERR_DSS_LINK_NOT_EXIST 2413 - +#define ERR_DSS_INIT_LOGGER_FAILED 2414 +#define ERR_DSS_OUT_OF_MEM 2415 +#define ERR_DSS_INVALID_ID 2416 +#define ERR_DSS_PROCESS_REMOTE 2417 +#define ERR_DSS_CONNECT_FAILED 2418 +#define ERR_DSS_VERSION_NOT_MATCH 2419 +#define ERR_DSS_INVALID_BLOCK_TYPE 2420 #define ERR_DSS_CEIL 2500 #endif // __DSS_API_DEF_H diff --git a/src/include/storage/smgr/smgr.h b/src/include/storage/smgr/smgr.h index 9acbb1643..5bb3af588 100644 --- a/src/include/storage/smgr/smgr.h +++ b/src/include/storage/smgr/smgr.h @@ -147,7 +147,7 @@ enum SMGR_READ_STATUS { * a pending fsync request getting canceled ... see mdsync). */ #ifndef WIN32 -#define FILE_POSSIBLY_DELETED(err) ((err) == ENOENT || (err) == ERR_DSS_DIR_NOT_EXIST || (err) == ERR_DSS_FILE_NOT_EXIST) +#define FILE_POSSIBLY_DELETED(err) ((err) == ENOENT || (err) == ERR_DSS_FILE_NOT_EXIST) #else #define FILE_POSSIBLY_DELETED(err) ((err) == ENOENT || (err) == EACCES) #endif From a1d3fbec09b08a85d5b1e2bba87b629c41df4f1a Mon Sep 17 00:00:00 2001 From: Julong-Li <584147810@qq.com> Date: Wed, 13 Sep 2023 10:05:47 +0800 Subject: [PATCH 228/304] =?UTF-8?q?issue:=20=E6=94=AF=E6=8C=81=E5=BD=93ope?= =?UTF-8?q?rator=E4=B8=A4=E7=AB=AF=E4=B8=BAvarchar=E4=B8=8Eint=E6=97=B6?= =?UTF-8?q?=EF=BC=8C=E5=B0=86=E4=B8=A4=E8=80=85=E5=BD=93=E4=BD=9Cnumeric?= =?UTF-8?q?=E6=9D=A5=E8=BF=9B=E8=A1=8C=E6=93=8D=E4=BD=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/gs_guc/cluster_guc.conf | 1 + src/common/backend/parser/parse_func.cpp | 4 ++-- src/common/backend/utils/misc/guc/guc_sql.cpp | 11 ++++++++++ .../knl/knl_guc/knl_session_attr_sql.h | 1 + .../expected/test_cast_in_operator.out | 21 +++++++++++++++++++ .../regress/output/recovery_2pc_tools.source | 1 + src/test/regress/parallel_schedule0 | 2 +- src/test/regress/parallel_schedule0A | 2 +- .../regress/sql/test_cast_in_operator.sql | 9 ++++++++ 9 files changed, 48 insertions(+), 4 deletions(-) create mode 100644 src/test/regress/expected/test_cast_in_operator.out create mode 100644 src/test/regress/sql/test_cast_in_operator.sql diff --git a/src/bin/gs_guc/cluster_guc.conf b/src/bin/gs_guc/cluster_guc.conf index 2ffd46d20..b52f72f7f 100755 --- a/src/bin/gs_guc/cluster_guc.conf +++ b/src/bin/gs_guc/cluster_guc.conf @@ -119,6 +119,7 @@ config_file|string|0,0|NULL|NULL| connection_alarm_rate|real|0,1|NULL|NULL| constraint_exclusion|enum|partition,on,off,true,false,yes,no,1,0|NULL|NULL| enable_union_all_subquery_orderby|bool|0,0|NULL|NULL| +transform_to_numeric_operators|bool|0,0|NULL|NULL| convert_string_to_digit|bool|0,0|NULL|Please don't modify this parameter which will change the type conversion rule and may lead to unpredictable behavior!| cost_param|int|0,2147483647|NULL|NULL| cpu_collect_timer|int|1,2147483647|NULL|NULL| diff --git a/src/common/backend/parser/parse_func.cpp b/src/common/backend/parser/parse_func.cpp index 523743c91..2bec3ceeb 100644 --- a/src/common/backend/parser/parse_func.cpp +++ b/src/common/backend/parser/parse_func.cpp @@ -1089,8 +1089,8 @@ FuncCandidateList func_select_candidate(int nargs, Oid* input_typeids, FuncCandi * so we should also admit highest type conversion for operations * between different type categories */ - if (u_sess->attr.attr_sql.sql_compatibility == C_FORMAT && !different_category && - slot_category[i] != TYPCATEGORY_UNKNOWN) { + if ((u_sess->attr.attr_sql.sql_compatibility == C_FORMAT || u_sess->attr.attr_sql.transform_to_numeric_operators) + && !different_category && slot_category[i] != TYPCATEGORY_UNKNOWN) { if (current_category == TYPCATEGORY_INVALID) current_category = slot_category[i]; else if (slot_category[i] != current_category) diff --git a/src/common/backend/utils/misc/guc/guc_sql.cpp b/src/common/backend/utils/misc/guc/guc_sql.cpp index e3111a23c..26115af26 100755 --- a/src/common/backend/utils/misc/guc/guc_sql.cpp +++ b/src/common/backend/utils/misc/guc/guc_sql.cpp @@ -1089,6 +1089,17 @@ static void InitSqlConfigureNamesBool() NULL, NULL, NULL}, + {{"transform_to_numeric_operators", + PGC_USERSET, + NODE_SINGLENODE, + QUERY_TUNING_METHOD, + gettext_noop("When turn on, choose numeric (op) numeric for varchar (op) int."), + NULL}, + &u_sess->attr.attr_sql.transform_to_numeric_operators, + false, + NULL, + NULL, + NULL}, {{"check_function_bodies", PGC_USERSET, NODE_ALL, diff --git a/src/include/knl/knl_guc/knl_session_attr_sql.h b/src/include/knl/knl_guc/knl_session_attr_sql.h index 74e69d666..537296e93 100644 --- a/src/include/knl/knl_guc/knl_session_attr_sql.h +++ b/src/include/knl/knl_guc/knl_session_attr_sql.h @@ -64,6 +64,7 @@ typedef struct knl_session_attr_sql { bool enable_bitmapscan; bool force_bitmapand; bool enable_union_all_subquery_orderby; + bool transform_to_numeric_operators; bool enable_parallel_ddl; bool enable_tidscan; bool enable_sort; diff --git a/src/test/regress/expected/test_cast_in_operator.out b/src/test/regress/expected/test_cast_in_operator.out new file mode 100644 index 000000000..65d22a7af --- /dev/null +++ b/src/test/regress/expected/test_cast_in_operator.out @@ -0,0 +1,21 @@ +set transform_to_numeric_operators = on; +create table cast_operator(a varchar); +insert into cast_operator values('5.6'); +insert into cast_operator values('5.6'); +insert into cast_operator values('3.6'); +insert into cast_operator values('2.6'); +select * from cast_operator where a>3; + a +----- + 5.6 + 5.6 + 3.6 +(3 rows) + +select count(*) from cast_operator group by a having a > 5; + count +------- + 2 +(1 row) + +drop table cast_operator; diff --git a/src/test/regress/output/recovery_2pc_tools.source b/src/test/regress/output/recovery_2pc_tools.source index 2fd58569c..0cd8b20ed 100644 --- a/src/test/regress/output/recovery_2pc_tools.source +++ b/src/test/regress/output/recovery_2pc_tools.source @@ -720,6 +720,7 @@ select name,vartype,unit,min_val,max_val from pg_settings where name <> 'qunit_c transaction_pending_time | integer | | -1 | 1073741823 transaction_read_only | bool | | | transform_null_equals | bool | | | + transform_to_numeric_operators | bool | | | transparent_encrypted_string | string | | | transparent_encrypt_kms_region | string | | | transparent_encrypt_kms_url | string | | | diff --git a/src/test/regress/parallel_schedule0 b/src/test/regress/parallel_schedule0 index c4abf02f2..3a3de2b2d 100644 --- a/src/test/regress/parallel_schedule0 +++ b/src/test/regress/parallel_schedule0 @@ -188,7 +188,7 @@ test: single_node_test_null_operator # ---------- # Another group of parallel tests # ---------- -test: single_node_create_aggregate +test: single_node_create_aggregate test_cast_in_operator #test: single_node_create_function_3 single_node_create_cast #test: single_node_constraints single_node_triggers single_node_inherit single_node_create_table_like single_node_typed_table test: single_node_vacuum diff --git a/src/test/regress/parallel_schedule0A b/src/test/regress/parallel_schedule0A index 1be0c5363..068fac93b 100644 --- a/src/test/regress/parallel_schedule0A +++ b/src/test/regress/parallel_schedule0A @@ -181,7 +181,7 @@ test: single_node_test_null_operator # ---------- # Another group of parallel tests # ---------- -test: single_node_create_aggregate +test: single_node_create_aggregate test_cast_in_operator #test: single_node_create_function_3 single_node_create_cast #test: single_node_constraints single_node_triggers single_node_inherit single_node_create_table_like single_node_typed_table test: single_node_vacuum diff --git a/src/test/regress/sql/test_cast_in_operator.sql b/src/test/regress/sql/test_cast_in_operator.sql new file mode 100644 index 000000000..25d160d4a --- /dev/null +++ b/src/test/regress/sql/test_cast_in_operator.sql @@ -0,0 +1,9 @@ +set transform_to_numeric_operators = on; +create table cast_operator(a varchar); +insert into cast_operator values('5.6'); +insert into cast_operator values('5.6'); +insert into cast_operator values('3.6'); +insert into cast_operator values('2.6'); +select * from cast_operator where a>3; +select count(*) from cast_operator group by a having a > 5; +drop table cast_operator; \ No newline at end of file From afa648b0d44bd97dc697d072aad5257432a40e54 Mon Sep 17 00:00:00 2001 From: WangXiuqiang Date: Wed, 13 Sep 2023 10:33:39 +0800 Subject: [PATCH 229/304] =?UTF-8?q?=E8=A7=A3=E5=86=B3sql=5Fmode=5Ffull=5Fg?= =?UTF-8?q?roup=E5=85=B3=E9=97=AD=E4=B8=8B=EF=BC=8C=E5=AF=B9=E6=B2=A1?= =?UTF-8?q?=E5=80=BC=E7=9A=84=E5=88=97=E4=BD=BF=E7=94=A8=E8=81=9A=E5=90=88?= =?UTF-8?q?=E5=87=BD=E6=95=B0=E6=8A=A5=E9=94=99=EF=BC=8C=E4=B8=8EMySQL?= =?UTF-8?q?=E8=BE=93=E5=87=BA=E4=B8=8D=E4=B8=80=E8=87=B4=E3=80=90zyzx?= =?UTF-8?q?=E3=80=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/storage/access/common/heaptuple.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/gausskernel/storage/access/common/heaptuple.cpp b/src/gausskernel/storage/access/common/heaptuple.cpp index 33bc78fc2..dbe468d82 100644 --- a/src/gausskernel/storage/access/common/heaptuple.cpp +++ b/src/gausskernel/storage/access/common/heaptuple.cpp @@ -1760,9 +1760,12 @@ void heap_slot_getsomeattrs(TupleTableSlot *slot, int attnum) } #endif - int attno = GetAttrNumber(slot, attnum); + int attno = attnum; + if (slot->tts_tuple) { + attno = GetAttrNumber(slot, attnum); - slot_deform_tuple(slot, attno); + slot_deform_tuple(slot, attno); + } /* If tuple doesn't have all the atts indicated by tupleDesc, read the rest as null */ if (unlikely(attno < attnum)) { From 59253c1f4a880d5608e16de637f8a28d8da64250 Mon Sep 17 00:00:00 2001 From: chenxiaobin19 <1025221611@qq.com> Date: Tue, 12 Sep 2023 20:58:32 +0800 Subject: [PATCH 230/304] =?UTF-8?q?A=E5=85=BC=E5=AE=B9=E6=80=A7=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E5=BA=93=E4=B8=8B=EF=BC=8C=E6=89=93=E5=BC=80char=5Fco?= =?UTF-8?q?erce=5Fcompat=E6=97=B6varchar=E7=9A=84=E6=AF=94=E8=BE=83?= =?UTF-8?q?=E5=8C=85=E5=90=AB=E7=A9=BA=E6=A0=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/parser/parse_oper.cpp | 10 + .../codegen/vecexecutor/vecexprcodegen.cpp | 8 + .../vecexecutor/vecprimitive/varchar.inl | 2 +- src/include/catalog/pg_operator.h | 3 + .../regress/expected/single_node_varchar.out | 233 ++++++++++++++++++ src/test/regress/sql/single_node_varchar.sql | 67 +++++ 6 files changed, 322 insertions(+), 1 deletion(-) diff --git a/src/common/backend/parser/parse_oper.cpp b/src/common/backend/parser/parse_oper.cpp index 2093e3c0f..292157e3d 100644 --- a/src/common/backend/parser/parse_oper.cpp +++ b/src/common/backend/parser/parse_oper.cpp @@ -402,6 +402,16 @@ Operator oper(ParseState* pstate, List* opname, Oid ltypeId, Oid rtypeId, bool n rtypeId = NUMERICOID; } + /* + * In A_FORMAT compatibility and CHAR_COERCE_COMPAT, we choose TEXT-related + * operators for varchar and bpchar. + */ + if (DB_IS_CMPT(A_FORMAT) && CHAR_COERCE_COMPAT && (((ltypeId == VARCHAROID) && (rtypeId == BPCHAROID)) || + ((rtypeId == VARCHAROID) && (ltypeId == BPCHAROID)))) { + ltypeId = TEXTOID; + rtypeId = TEXTOID; + } + /* * Try to find the mapping in the lookaside cache. */ diff --git a/src/gausskernel/runtime/codegen/vecexecutor/vecexprcodegen.cpp b/src/gausskernel/runtime/codegen/vecexecutor/vecexprcodegen.cpp index a0f0e5f06..eaa4c0f66 100644 --- a/src/gausskernel/runtime/codegen/vecexecutor/vecexprcodegen.cpp +++ b/src/gausskernel/runtime/codegen/vecexecutor/vecexprcodegen.cpp @@ -658,8 +658,16 @@ bool VecExprCodeGen::OpJittable(ExprState* state) return false; switch (opexpr->opno) { + case TEXTEQOID: + case TEXTNEOID: + case TEXTLEOID: + case TEXTGEOID: case TEXTLTOID: case TEXTGTOID: { + if (DB_IS_CMPT(A_FORMAT) && CHAR_COERCE_COMPAT) { + return false; + } + /* Only support ASCII and UTF-8 encoding */ int current_encoding = GetDatabaseEncoding(); if (current_encoding != PG_SQL_ASCII && current_encoding != PG_UTF8) diff --git a/src/gausskernel/runtime/vecexecutor/vecprimitive/varchar.inl b/src/gausskernel/runtime/vecexecutor/vecprimitive/varchar.inl index 9d60cca42..96b8cc3bb 100755 --- a/src/gausskernel/runtime/vecexecutor/vecprimitive/varchar.inl +++ b/src/gausskernel/runtime/vecexecutor/vecprimitive/varchar.inl @@ -765,7 +765,7 @@ vtrim1(PG_FUNCTION_ARGS) FunctionCallInfoData finfo; finfo.arg = &args[0]; - + finfo.flinfo = fcinfo->flinfo; if (pselection != NULL) { diff --git a/src/include/catalog/pg_operator.h b/src/include/catalog/pg_operator.h index d03916cf9..89e94322f 100644 --- a/src/include/catalog/pg_operator.h +++ b/src/include/catalog/pg_operator.h @@ -165,8 +165,11 @@ typedef FormData_pg_operator *Form_pg_operator; #define INT1EQOID 5513 #define OID_NAME_REGEXEQ_OP 639 #define OID_TEXT_REGEXEQ_OP 641 +#define TEXTNEOID 531 #define TEXTLTOID 664 +#define TEXTLEOID 665 #define TEXTGTOID 666 +#define TEXTGEOID 667 #define FLOAT8EQOID 670 #define FLOAT8NEOID 671 #define FLOAT8LTOID 672 diff --git a/src/test/regress/expected/single_node_varchar.out b/src/test/regress/expected/single_node_varchar.out index dae59316b..d5c2d9343 100644 --- a/src/test/regress/expected/single_node_varchar.out +++ b/src/test/regress/expected/single_node_varchar.out @@ -154,3 +154,236 @@ select length(rtrim('123 ')); (1 row) set behavior_compat_options = ''; +set behavior_compat_options = 'char_coerce_compat'; +create table tbl_111 (id int); +insert into tbl_111 values(1); +select count(*) from tbl_111 where cast(' ' as char(5)) = cast(' ' as varchar(5)); + count +------- + 0 +(1 row) + +select count(*) from tbl_111 where cast('abc ' as char(10)) = cast('abc ' as varchar(10)); + count +------- + 0 +(1 row) + +select count(*) from tbl_111 where cast(' ' as char(5)) != cast(' ' as varchar(5)); + count +------- + 1 +(1 row) + +select count(*) from tbl_111 where cast('abc ' as char(10)) != cast('abc ' as varchar(10)); + count +------- + 1 +(1 row) + +select count(*) from tbl_111 where cast(' ' as char(5)) > cast(' ' as varchar(5)); + count +------- + 1 +(1 row) + +select count(*) from tbl_111 where cast('abc ' as char(10)) > cast('abc ' as varchar(10)); + count +------- + 1 +(1 row) + +select count(*) from tbl_111 where cast(' ' as char(5)) >= cast(' ' as varchar(5)); + count +------- + 1 +(1 row) + +select count(*) from tbl_111 where cast('abc ' as char(10)) >= cast('abc ' as varchar(10)); + count +------- + 1 +(1 row) + +select count(*) from tbl_111 where cast(' ' as char(5)) < cast(' ' as varchar(5)); + count +------- + 0 +(1 row) + +select count(*) from tbl_111 where cast('abc ' as char(10)) < cast('abc ' as varchar(10)); + count +------- + 0 +(1 row) + +select count(*) from tbl_111 where cast(' ' as char(5)) <= cast(' ' as varchar(5)); + count +------- + 0 +(1 row) + +select count(*) from tbl_111 where cast('abc ' as char(10)) <= cast('abc ' as varchar(10)); + count +------- + 0 +(1 row) + +drop table tbl_111; +create table tbl_111 (a char(5), b varchar(5)); +insert into tbl_111 values (' ', ' '); +insert into tbl_111 values ('abc ', 'abc '); +select count(*) from tbl_111 where a = b; + count +------- + 0 +(1 row) + +select count(*) from tbl_111 where a != b; + count +------- + 2 +(1 row) + +select count(*) from tbl_111 where a > b; + count +------- + 2 +(1 row) + +select count(*) from tbl_111 where a >= b; + count +------- + 2 +(1 row) + +select count(*) from tbl_111 where a < b; + count +------- + 0 +(1 row) + +select count(*) from tbl_111 where a <= b; + count +------- + 0 +(1 row) + +set try_vector_engine_strategy='force'; +select count(*) from tbl_111 where a = b; + count +------- + 0 +(1 row) + +select count(*) from tbl_111 where a != b; + count +------- + 2 +(1 row) + +select count(*) from tbl_111 where a > b; + count +------- + 2 +(1 row) + +select count(*) from tbl_111 where a >= b; + count +------- + 2 +(1 row) + +select count(*) from tbl_111 where a < b; + count +------- + 0 +(1 row) + +select count(*) from tbl_111 where a <= b; + count +------- + 0 +(1 row) + +set try_vector_engine_strategy='off'; +drop table tbl_111; +create table tbl_111 (a char(5), b varchar(5)) with (orientation = column); +insert into tbl_111 values (' ', ' '); +insert into tbl_111 values ('abc ', 'abc '); +select count(*) from tbl_111 where a = b; + count +------- + 0 +(1 row) + +select count(*) from tbl_111 where a != b; + count +------- + 2 +(1 row) + +select count(*) from tbl_111 where a > b; + count +------- + 2 +(1 row) + +select count(*) from tbl_111 where a >= b; + count +------- + 2 +(1 row) + +select count(*) from tbl_111 where a < b; + count +------- + 0 +(1 row) + +select count(*) from tbl_111 where a <= b; + count +------- + 0 +(1 row) + +set enable_codegen to true; +set codegen_cost_threshold to 0; +select count(*) from tbl_111 where a = b; + count +------- + 0 +(1 row) + +select count(*) from tbl_111 where a != b; + count +------- + 2 +(1 row) + +select count(*) from tbl_111 where a > b; + count +------- + 2 +(1 row) + +select count(*) from tbl_111 where a >= b; + count +------- + 2 +(1 row) + +select count(*) from tbl_111 where a < b; + count +------- + 0 +(1 row) + +select count(*) from tbl_111 where a <= b; + count +------- + 0 +(1 row) + +drop table tbl_111; +set behavior_compat_options = ''; diff --git a/src/test/regress/sql/single_node_varchar.sql b/src/test/regress/sql/single_node_varchar.sql index 0147ae336..48e604499 100644 --- a/src/test/regress/sql/single_node_varchar.sql +++ b/src/test/regress/sql/single_node_varchar.sql @@ -85,3 +85,70 @@ select length(rtrim('123 ')); set behavior_compat_options = 'char_coerce_compat'; select length(rtrim('123 ')); set behavior_compat_options = ''; + +set behavior_compat_options = 'char_coerce_compat'; +create table tbl_111 (id int); +insert into tbl_111 values(1); + +select count(*) from tbl_111 where cast(' ' as char(5)) = cast(' ' as varchar(5)); +select count(*) from tbl_111 where cast('abc ' as char(10)) = cast('abc ' as varchar(10)); + +select count(*) from tbl_111 where cast(' ' as char(5)) != cast(' ' as varchar(5)); +select count(*) from tbl_111 where cast('abc ' as char(10)) != cast('abc ' as varchar(10)); + +select count(*) from tbl_111 where cast(' ' as char(5)) > cast(' ' as varchar(5)); +select count(*) from tbl_111 where cast('abc ' as char(10)) > cast('abc ' as varchar(10)); + +select count(*) from tbl_111 where cast(' ' as char(5)) >= cast(' ' as varchar(5)); +select count(*) from tbl_111 where cast('abc ' as char(10)) >= cast('abc ' as varchar(10)); + +select count(*) from tbl_111 where cast(' ' as char(5)) < cast(' ' as varchar(5)); +select count(*) from tbl_111 where cast('abc ' as char(10)) < cast('abc ' as varchar(10)); + +select count(*) from tbl_111 where cast(' ' as char(5)) <= cast(' ' as varchar(5)); +select count(*) from tbl_111 where cast('abc ' as char(10)) <= cast('abc ' as varchar(10)); + +drop table tbl_111; + +create table tbl_111 (a char(5), b varchar(5)); +insert into tbl_111 values (' ', ' '); +insert into tbl_111 values ('abc ', 'abc '); +select count(*) from tbl_111 where a = b; +select count(*) from tbl_111 where a != b; +select count(*) from tbl_111 where a > b; +select count(*) from tbl_111 where a >= b; +select count(*) from tbl_111 where a < b; +select count(*) from tbl_111 where a <= b; + +set try_vector_engine_strategy='force'; +select count(*) from tbl_111 where a = b; +select count(*) from tbl_111 where a != b; +select count(*) from tbl_111 where a > b; +select count(*) from tbl_111 where a >= b; +select count(*) from tbl_111 where a < b; +select count(*) from tbl_111 where a <= b; +set try_vector_engine_strategy='off'; + +drop table tbl_111; + +create table tbl_111 (a char(5), b varchar(5)) with (orientation = column); +insert into tbl_111 values (' ', ' '); +insert into tbl_111 values ('abc ', 'abc '); +select count(*) from tbl_111 where a = b; +select count(*) from tbl_111 where a != b; +select count(*) from tbl_111 where a > b; +select count(*) from tbl_111 where a >= b; +select count(*) from tbl_111 where a < b; +select count(*) from tbl_111 where a <= b; + +set enable_codegen to true; +set codegen_cost_threshold to 0; +select count(*) from tbl_111 where a = b; +select count(*) from tbl_111 where a != b; +select count(*) from tbl_111 where a > b; +select count(*) from tbl_111 where a >= b; +select count(*) from tbl_111 where a < b; +select count(*) from tbl_111 where a <= b; + +drop table tbl_111; +set behavior_compat_options = ''; From e100f9596789d096d319167b1f68a7a59eaeb1dc Mon Sep 17 00:00:00 2001 From: kenxx Date: Wed, 13 Sep 2023 03:23:46 -0400 Subject: [PATCH 231/304] not allow to drop mlog --- src/common/backend/catalog/gs_matview.cpp | 36 ++++++++++++++++++ src/common/backend/catalog/heap.cpp | 4 +- .../optimizer/commands/tablecmds.cpp | 8 ++++ src/include/catalog/gs_matview.h | 2 + src/test/regress/expected/matview_single.out | 37 +++++++++++++++++++ src/test/regress/sql/matview_single.sql | 33 +++++++++++++++++ 6 files changed, 118 insertions(+), 2 deletions(-) diff --git a/src/common/backend/catalog/gs_matview.cpp b/src/common/backend/catalog/gs_matview.cpp index 5f479e608..dadfbf1d6 100644 --- a/src/common/backend/catalog/gs_matview.cpp +++ b/src/common/backend/catalog/gs_matview.cpp @@ -340,6 +340,42 @@ void delete_matdep_table(Oid mlogid) return; } +Oid get_matview_mlog_baserelid(Oid mlogOid) +{ + Relation relation = NULL; + TableScanDesc scan; + ScanKeyData scanKey; + HeapTuple tup = NULL; + Datum relid = 0; + Oid baserelid = InvalidOid; + bool isnull; + + ScanKeyInit(&scanKey, + Anum_gs_matview_dep_mlogid, + BTEqualStrategyNumber, + F_OIDEQ, + ObjectIdGetDatum(mlogOid)); + relation = heap_open(MatviewDependencyId, AccessShareLock); + scan = tableam_scan_begin(relation, SnapshotNow, 1, &scanKey); + + while((tup = (HeapTuple) tableam_scan_getnexttuple(scan, ForwardScanDirection)) != NULL) { + relid = heap_getattr(tup, + Anum_gs_matview_dep_relid, + RelationGetDescr(relation), + &isnull); + if (!isnull) { + baserelid = DatumGetObjectId(relid); + /* find out baserelid*/ + break; + } + } + + tableam_scan_end(scan); + heap_close(relation, NoLock); + + return baserelid; +} + Datum get_matview_refreshtime(Oid matviewOid, bool *isNULL) { Relation relation = NULL; diff --git a/src/common/backend/catalog/heap.cpp b/src/common/backend/catalog/heap.cpp index 905451180..9f58d2337 100644 --- a/src/common/backend/catalog/heap.cpp +++ b/src/common/backend/catalog/heap.cpp @@ -42,6 +42,7 @@ #include "access/multixact.h" #include "catalog/catalog.h" #include "catalog/dependency.h" +#include "catalog/gs_matview.h" #include "catalog/gs_obsscaninfo.h" #include "catalog/heap.h" #include "catalog/index.h" @@ -3695,8 +3696,7 @@ void heap_drop_with_catalog(Oid relid) * something with the doomed relation. */ if (ISMLOG(RelationGetForm(rel)->relname.data)) { - char *base_relid_str = RelationGetForm(rel)->relname.data + MLOGLEN; - Oid base_relid = atoi(base_relid_str); + Oid base_relid = get_matview_mlog_baserelid(relid); if (OidIsValid(base_relid)) { CacheInvalidateRelcacheByRelid(base_relid); } diff --git a/src/gausskernel/optimizer/commands/tablecmds.cpp b/src/gausskernel/optimizer/commands/tablecmds.cpp index f5b9d406e..b24bcf71a 100755 --- a/src/gausskernel/optimizer/commands/tablecmds.cpp +++ b/src/gausskernel/optimizer/commands/tablecmds.cpp @@ -37,6 +37,7 @@ #include "access/multixact.h" #include "catalog/catalog.h" #include "catalog/dependency.h" +#include "catalog/gs_matview.h" #include "catalog/heap.h" #include "catalog/index.h" #include "catalog/indexing.h" @@ -3664,6 +3665,13 @@ void RemoveRelations(DropStmt* drop, StringInfo tmp_queryString, RemoteQueryExec } delrel = try_relation_open(relOid, NoLock); + /*Not allow to drop mlog*/ + if (relkind == RELKIND_RELATION && delrel != NULL && ISMLOG(delrel->rd_rel->relname.data)) { + /*If we can find a base table, it is mlog.*/ + if (get_matview_mlog_baserelid(relOid)!= InvalidOid) + ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("Use 'Drop table' to drop mlog table %s is not allowed.",delrel->rd_rel->relname.data))); + } /* * Open up drop table command for table being redistributed right now. * diff --git a/src/include/catalog/gs_matview.h b/src/include/catalog/gs_matview.h index 72654de3f..0aa554eb9 100644 --- a/src/include/catalog/gs_matview.h +++ b/src/include/catalog/gs_matview.h @@ -86,4 +86,6 @@ extern void acquire_mativew_tables_lock(Query *query, bool incremental); extern bool CheckMatviewQuals(Query *query); extern Oid FindRoleid(Oid relid); +extern Oid get_matview_mlog_baserelid(Oid mlogOid); + #endif /* GS_MATVIEW_H */ diff --git a/src/test/regress/expected/matview_single.out b/src/test/regress/expected/matview_single.out index 4d3c4042e..edf513a5a 100644 --- a/src/test/regress/expected/matview_single.out +++ b/src/test/regress/expected/matview_single.out @@ -141,6 +141,43 @@ drop table test_syn cascade; NOTICE: drop cascades to 2 other objects DETAIL: drop cascades to materialized view mv_test_syn drop cascades to materialized view imv_test_syn +-- case 5: drop mlog table. +create table imv1_t(a int); +insert into imv1_t values(1); +create incremental materialized view imv1_v as select * from imv1_t; +declare + oid int := (select oid from pg_class where relname = 'imv1_t'); + table_name varchar(20) := 'mlog_' || oid; + sql_stmt text := 'Drop table ' || table_name; +begin + execute sql_stmt; +END; +/ +--?ERROR: Use 'Drop table' to drop mlog table mlog_.* is not allowed. +--?CONTEXT: SQL statement "Drop table mlog_.*" +PL/pgSQL function inline_code_block line 5 at EXECUTE statement +-- case 6: drop table that looks like a mlog with valid oid. +drop materialized view imv1_v; +declare + oid int := (select oid from pg_class where relname = 'imv1_t'); + table_name varchar(20) := 'mlog_' || oid; + create_stmt text := 'Create table ' || table_name || '(a int)' ; + drop_stmt text := 'Drop table ' || table_name; +begin + execute create_stmt; + execute drop_stmt; +END; +/ +--?WARNING: "mlog_.*" is not an appropriated name for relation +DETAIL: The kernel may treat it as a mlog table of materialized view +--?CONTEXT: SQL statement "Create table mlog_.*(a int)" +PL/pgSQL function inline_code_block line 6 at EXECUTE statement +-- case 7: drop table that looks like a mlog without valid oid. +create table mlog_99999(a int); +WARNING: "mlog_99999" is not an appropriated name for relation +DETAIL: The kernel may treat it as a mlog table of materialized view +drop table mlog_99999; +drop table imv1_t cascade; -- test about the privileges of refresh create table t (id int); insert into t select generate_series(1,10); diff --git a/src/test/regress/sql/matview_single.sql b/src/test/regress/sql/matview_single.sql index 6f1502b9d..c9adb65d2 100644 --- a/src/test/regress/sql/matview_single.sql +++ b/src/test/regress/sql/matview_single.sql @@ -72,6 +72,39 @@ drop synonym s_mv_test_syn; drop synonym s_imv_test_syn; drop table test_syn cascade; +-- case 5: drop mlog table. +create table imv1_t(a int); +insert into imv1_t values(1); +create incremental materialized view imv1_v as select * from imv1_t; + +declare + oid int := (select oid from pg_class where relname = 'imv1_t'); + table_name varchar(20) := 'mlog_' || oid; + sql_stmt text := 'Drop table ' || table_name; +begin + execute sql_stmt; +END; +/ + +-- case 6: drop table that looks like a mlog with valid oid. +drop materialized view imv1_v; +declare + oid int := (select oid from pg_class where relname = 'imv1_t'); + table_name varchar(20) := 'mlog_' || oid; + create_stmt text := 'Create table ' || table_name || '(a int)' ; + drop_stmt text := 'Drop table ' || table_name; +begin + execute create_stmt; + execute drop_stmt; +END; +/ + +-- case 7: drop table that looks like a mlog without valid oid. +create table mlog_99999(a int); +drop table mlog_99999; + +drop table imv1_t cascade; + -- test about the privileges of refresh create table t (id int); insert into t select generate_series(1,10); From a0e4d6cf1ad39d1481bed3929cb1f956e1e61a58 Mon Sep 17 00:00:00 2001 From: teooooozhang Date: Wed, 13 Sep 2023 15:31:48 +0800 Subject: [PATCH 232/304] =?UTF-8?q?issue=E4=BF=AE=E5=A4=8D=EF=BC=9Ags=5Fba?= =?UTF-8?q?sebackup=E5=9B=A0=E6=96=B0=E5=A2=9E=E5=8F=82=E6=95=B0=E6=A0=A1?= =?UTF-8?q?=E9=AA=8C=E5=AF=BC=E8=87=B4=E9=87=8D=E5=AE=9A=E5=90=91=E5=8A=9F?= =?UTF-8?q?=E8=83=BD=E5=A4=B1=E6=95=88=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/pg_basebackup/pg_basebackup.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/bin/pg_basebackup/pg_basebackup.cpp b/src/bin/pg_basebackup/pg_basebackup.cpp index 66540c0a9..9a84d71e4 100644 --- a/src/bin/pg_basebackup/pg_basebackup.cpp +++ b/src/bin/pg_basebackup/pg_basebackup.cpp @@ -1960,13 +1960,11 @@ static int GsBaseBackup(int argc, char** argv) int i; for (i = 0; i < argc; i++) { char *optstr = argv[i]; - int is_only_shortbar; - if (strlen(optstr) == 1) { - is_only_shortbar = optstr[0] == '-' ? 1 : 0; - } else { - is_only_shortbar = 0; - } - if (is_only_shortbar) { + if (strlen(optstr) == 1 && optstr[0] == '-') { + /* ignore the case of redirecting output like "gs_probackup ... -D -> xxx.tar.gz" */ + if (i > 0 && strcmp(argv[i - 1], "-D") == 0) { + continue; + } fprintf(stderr, _("%s: The option '-' is not a valid option.\n"), progname); exit(1); } @@ -1986,9 +1984,11 @@ static int GsBaseBackup(int argc, char** argv) } char *next_optstr = argv[i + 1]; + if (strcmp(optstr, "-D") == 0 && strcmp(next_optstr, "-") == 0) { + continue; + } char *next_oli = strchr(optstring, next_optstr[1]); - int is_arg_optionform = next_optstr[0] == '-' && next_oli != NULL; - if (is_arg_optionform) { + if (next_optstr[0] == '-' && next_oli != NULL) { fprintf(stderr, _("%s: The option '-%c' need a parameter.\n"), progname, optstr[1]); exit(1); } From f759a1ceefe4117730e15d3311ead77d0b89392c Mon Sep 17 00:00:00 2001 From: "arcoalien@qq.com" Date: Wed, 13 Sep 2023 16:38:21 +0800 Subject: [PATCH 233/304] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E5=85=B3=E9=97=AD?= =?UTF-8?q?=E5=8F=8C=E5=86=99=E5=90=8E=EF=BC=8C=E6=AE=B5=E9=A1=B5=E5=BC=8F?= =?UTF-8?q?=E6=89=A7=E8=A1=8Cgs=5Fshrink=E4=BC=9Acore=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../storage/smgr/segment/segxlog.cpp | 29 +++++++++++-------- .../storage/smgr/segment/space.cpp | 27 +++++++++-------- 2 files changed, 32 insertions(+), 24 deletions(-) diff --git a/src/gausskernel/storage/smgr/segment/segxlog.cpp b/src/gausskernel/storage/smgr/segment/segxlog.cpp index 264c8bb86..8efa9fa38 100644 --- a/src/gausskernel/storage/smgr/segment/segxlog.cpp +++ b/src/gausskernel/storage/smgr/segment/segxlog.cpp @@ -780,20 +780,25 @@ void seg_redo_new_page_copy_and_flush(BufferTag *tag, char *data, XLogRecPtr lsn update_max_page_flush_lsn(lsn, t_thrd.proc_cxt.MyProcPid, false); } - bool flush_old_file = false; - uint32 pos = seg_dw_single_flush_without_buffer(*tag, (Block)page, &flush_old_file); - t_thrd.proc->dw_pos = pos; - t_thrd.proc->flush_new_dw = !flush_old_file; - - SegSpace *spc = spc_open(tag->rnode.spcNode, tag->rnode.dbNode, false); - SegmentCheck(spc != NULL); - seg_physical_write(spc, tag->rnode, tag->forkNum, tag->blockNum, page, true); - if (flush_old_file) { - g_instance.dw_single_cxt.recovery_buf.single_flush_state[pos] = true; + if (dw_enabled() && pg_atomic_read_u32(&g_instance.ckpt_cxt_ctl->current_page_writer_count) > 0) { + bool flush_old_file = false; + uint16 pos = seg_dw_single_flush_without_buffer(*tag, (Block)page, &flush_old_file); + t_thrd.proc->dw_pos = pos; + t_thrd.proc->flush_new_dw = !flush_old_file; + SegSpace *spc = spc_open(tag->rnode.spcNode, tag->rnode.dbNode, false); + SegmentCheck(spc != NULL); + seg_physical_write(spc, tag->rnode, tag->forkNum, tag->blockNum, page, true); + if (flush_old_file) { + g_instance.dw_single_cxt.recovery_buf.single_flush_state[pos] = true; + } else { + g_instance.dw_single_cxt.single_flush_state[pos] = true; + } + t_thrd.proc->dw_pos = -1; } else { - g_instance.dw_single_cxt.single_flush_state[pos] = true; + SegSpace *spc = spc_open(tag->rnode.spcNode, tag->rnode.dbNode, false); + SegmentCheck(spc != NULL); + seg_physical_write(spc, tag->rnode, tag->forkNum, tag->blockNum, page, true); } - t_thrd.proc->dw_pos = -1; } diff --git a/src/gausskernel/storage/smgr/segment/space.cpp b/src/gausskernel/storage/smgr/segment/space.cpp index 122ff7884..be034f835 100644 --- a/src/gausskernel/storage/smgr/segment/space.cpp +++ b/src/gausskernel/storage/smgr/segment/space.cpp @@ -582,20 +582,23 @@ static void copy_extent(SegExtentGroup *seg, RelFileNode logic_rnode, uint32 log PageSetLSN(pagedata, recptr); /* 2. double write */ - bool flush_old_file = false; - uint32 pos = seg_dw_single_flush_without_buffer(tag, (Block)pagedata, &flush_old_file); - t_thrd.proc->dw_pos = pos; - t_thrd.proc->flush_new_dw = !flush_old_file; - - /* 3. checksum and write to file */ - PageSetChecksumInplace((Page)pagedata, to_block); - df_pwrite_block(seg->segfile, pagedata, to_block); - if (flush_old_file) { - g_instance.dw_single_cxt.recovery_buf.single_flush_state[pos] = true; + if (dw_enabled() && pg_atomic_read_u32(&g_instance.ckpt_cxt_ctl->current_page_writer_count) > 0) { + bool flush_old_file = false; + uint16 pos = seg_dw_single_flush_without_buffer(tag, (Block)pagedata, &flush_old_file); + t_thrd.proc->dw_pos = pos; + t_thrd.proc->flush_new_dw = !flush_old_file; + PageSetChecksumInplace((Page)pagedata, to_block); + df_pwrite_block(seg->segfile, pagedata, to_block); + if (flush_old_file) { + g_instance.dw_single_cxt.recovery_buf.single_flush_state[pos] = true; + } else { + g_instance.dw_single_cxt.single_flush_state[pos] = true; + } + t_thrd.proc->dw_pos = -1; } else { - g_instance.dw_single_cxt.single_flush_state[pos] = true; + PageSetChecksumInplace((Page)pagedata, to_block); + df_pwrite_block(seg->segfile, pagedata, to_block); } - t_thrd.proc->dw_pos = -1; } END_CRIT_SECTION(); From 1c72e782ffdf359684848583c79b630d5f6784a2 Mon Sep 17 00:00:00 2001 From: l30039603 Date: Wed, 13 Sep 2023 17:05:50 +0800 Subject: [PATCH 234/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=B5=8C=E5=A5=97?= =?UTF-8?q?=E5=87=BD=E6=95=B0step=E4=B8=8Econtinue=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/pl/plpgsql/src/pl_debugger.cpp | 2 +- .../regress/expected/pl_debugger_client.out | 70 ++----------------- src/test/regress/sql/pl_debugger_client.sql | 12 ---- 3 files changed, 5 insertions(+), 79 deletions(-) diff --git a/src/common/pl/plpgsql/src/pl_debugger.cpp b/src/common/pl/plpgsql/src/pl_debugger.cpp index d2676b8b5..e931fc454 100644 --- a/src/common/pl/plpgsql/src/pl_debugger.cpp +++ b/src/common/pl/plpgsql/src/pl_debugger.cpp @@ -190,7 +190,7 @@ void check_debug(PLpgSQL_function* func, PLpgSQL_execstate* estate) /* maintain session's debug server is on base turn on function */ u_sess->plsql_cxt.cur_debug_server = func->debug; } - func->debug->stop_next_stmt = need_continue_into ? false : true; + func->debug->stop_next_stmt = (need_continue_into && !is_stepinto) ? false : true; } } diff --git a/src/test/regress/expected/pl_debugger_client.out b/src/test/regress/expected/pl_debugger_client.out index 9d994e05a..e46163a37 100755 --- a/src/test/regress/expected/pl_debugger_client.out +++ b/src/test/regress/expected/pl_debugger_client.out @@ -661,54 +661,12 @@ select * from tmp_holder; 2:12: raise info 'pi_return : %',pi_return ; (6 rows) -select funcname, lineno, query from dbe_pldebugger.continue(); - funcname | lineno | query -------------+--------+--------------- - test_debug | 15 | cnt := 0; -(1 row) - -select * from dbe_pldebugger.info_locals(); - varname | vartype | value | package_name | isconst ------------+-----------+----------------------------+--------------+--------- - x | int4 | 1 | | f - sql_stmt | varchar | | | f - test.a | int4 | 0 | | f - test.b | varchar | | | f - test.c | timestamp | Sat Jan 01 00:00:00 2000 | | f - r | Row | [ (null), (null), (null),] | | f - rec | Rec | | | f - b_tmp | text | | | f - cnt | int4 | 0 | | f - a_tmp | int4 | 0 | | f - cur | refcursor | {name: +| | f - | | is_open: f +| | - | | found: f +| | - | | not_found: f +| | - | | row_count: 0} | | - n_tmp | numeric | | | f - t_tmp | tsquery | | | f - criterion | int4 | 0 | | f - cur_arg | refcursor | {name: +| | f - | | is_open: f +| | - | | found: f +| | - | | not_found: f +| | - | | row_count: 0} | | - index_1 | int4 | 0 | | f -(16 rows) - select funcname, lineno, query from dbe_pldebugger.continue(); funcname | lineno | query ------------+--------+------------------------------------------------ test_debug | 31 | SELECT b FROM test where a = 7 INTO b_tmp; (1 row) -select frameno, funcname, lineno, query from dbe_pldebugger.backtrace(); - frameno | funcname | lineno | query ----------+-------------+--------+------------------------------------------------ - 0 | test_debug | 31 | SELECT b FROM test where a = 7 INTO b_tmp; - 1 | test_debug4 | 7 | call test_debug(a); -(2 rows) - select * from dbe_pldebugger.info_locals(); varname | vartype | value | package_name | isconst -----------+-----------+--------------------------------------------------------+--------------+--------- @@ -859,12 +817,6 @@ begin insert into tmp_holder select breakpointno || ':' || lineno || ':' || query from dbe_pldebugger.info_breakpoints(); end; $$; -select funcname, lineno, query from dbe_pldebugger.finish(); - funcname | lineno | query -------------+--------+--------------- - test_debug | 15 | cnt := 0; -(1 row) - select funcname, lineno, query from dbe_pldebugger.finish(); funcname | lineno | query ------------+--------+------------------------------------------------ @@ -956,20 +908,6 @@ select frameno, funcname, lineno, query from dbe_pldebugger.backtrace(); 1 | test_debug_recursive | 5 | return query select * from test_debug_recursive(ct+ 1, pr * (ct+ 1)); (2 rows) -select funcname, lineno, query from dbe_pldebugger.continue(); - funcname | lineno | query -----------------------+--------+--------------------------------- - test_debug_recursive | 3 | return query select ct, pr; -(1 row) - -select frameno, funcname, lineno, query from dbe_pldebugger.backtrace(); - frameno | funcname | lineno | query ----------+----------------------+--------+------------------------------------------------------------------------------- - 0 | test_debug_recursive | 3 | return query select ct, pr; - 1 | test_debug_recursive | 5 | return query select * from test_debug_recursive(ct+ 1, pr * (ct+ 1)); - 2 | test_debug_recursive | 5 | return query select * from test_debug_recursive(ct+ 1, pr * (ct+ 1)); -(3 rows) - select funcname, lineno, query from dbe_pldebugger.continue(); funcname | lineno | query ----------------------+--------+------------------------------------------------------------------------------- @@ -985,15 +923,15 @@ select frameno, funcname, lineno, query from dbe_pldebugger.backtrace(); (3 rows) select funcname, lineno, query from dbe_pldebugger.continue(); - funcname | lineno | query -----------------------+--------+--------------------------------- - test_debug_recursive | 3 | return query select ct, pr; + funcname | lineno | query +----------------------+--------+------------------------------------------------------------------------------- + test_debug_recursive | 5 | return query select * from test_debug_recursive(ct+ 1, pr * (ct+ 1)); (1 row) select frameno, funcname, lineno, query from dbe_pldebugger.backtrace(); frameno | funcname | lineno | query ---------+----------------------+--------+------------------------------------------------------------------------------- - 0 | test_debug_recursive | 3 | return query select ct, pr; + 0 | test_debug_recursive | 5 | return query select * from test_debug_recursive(ct+ 1, pr * (ct+ 1)); 1 | test_debug_recursive | 5 | return query select * from test_debug_recursive(ct+ 1, pr * (ct+ 1)); 2 | test_debug_recursive | 5 | return query select * from test_debug_recursive(ct+ 1, pr * (ct+ 1)); 3 | test_debug_recursive | 5 | return query select * from test_debug_recursive(ct+ 1, pr * (ct+ 1)); diff --git a/src/test/regress/sql/pl_debugger_client.sql b/src/test/regress/sql/pl_debugger_client.sql index 1e2873fe8..3f0034531 100644 --- a/src/test/regress/sql/pl_debugger_client.sql +++ b/src/test/regress/sql/pl_debugger_client.sql @@ -218,12 +218,6 @@ select * from dbe_pldebugger.info_locals(); select funcname, lineno, query from dbe_pldebugger.continue(); -select frameno, funcname, lineno, query from dbe_pldebugger.backtrace(); - -select * from dbe_pldebugger.info_locals(); - -select funcname, lineno, query from dbe_pldebugger.continue(); - select funcname, lineno, query from dbe_pldebugger.continue(); -- test with finish without encountered breakpoint @@ -269,8 +263,6 @@ select funcname, lineno, query from dbe_pldebugger.finish(); select funcname, lineno, query from dbe_pldebugger.finish(); -select funcname, lineno, query from dbe_pldebugger.finish(); - -- test recursive debug select pg_sleep(1); @@ -313,10 +305,6 @@ select funcname, lineno, query from dbe_pldebugger.continue(); select frameno, funcname, lineno, query from dbe_pldebugger.backtrace(); -select funcname, lineno, query from dbe_pldebugger.continue(); - -select frameno, funcname, lineno, query from dbe_pldebugger.backtrace(); - select * from dbe_pldebugger.info_locals(); select * from dbe_pldebugger.info_locals(0); From dd8d38c875049b47fdd3d7702358b0bbe6a66143 Mon Sep 17 00:00:00 2001 From: totaj Date: Wed, 13 Sep 2023 21:06:23 +0800 Subject: [PATCH 235/304] Fix on update with trigger. --- .../runtime/executor/nodeModifyTable.cpp | 1 + .../regress/expected/single_node_update.out | 39 +++++++++++++++++++ src/test/regress/sql/single_node_update.sql | 39 +++++++++++++++++++ 3 files changed, 79 insertions(+) diff --git a/src/gausskernel/runtime/executor/nodeModifyTable.cpp b/src/gausskernel/runtime/executor/nodeModifyTable.cpp index c78fd44ed..924f3b6cf 100644 --- a/src/gausskernel/runtime/executor/nodeModifyTable.cpp +++ b/src/gausskernel/runtime/executor/nodeModifyTable.cpp @@ -511,6 +511,7 @@ bool ExecComputeStoredUpdateExpr(ResultRelInfo *resultRelInfo, EState *estate, T attnum = bms_next_member(updatedCols, temp_id); updated_colnum_resno = attnum + FirstLowInvalidHeapAttributeNumber; temp_id = attnum; + tableam_tslot_getallattrs(slot); for (int32 i = 0; i < (int32)natts; i++) { if (updated_colnum_resno == (uint32)(i + 1)) { if (slot->tts_isnull[i] && oldnulls[i]) { diff --git a/src/test/regress/expected/single_node_update.out b/src/test/regress/expected/single_node_update.out index a334ae5b4..662753fad 100644 --- a/src/test/regress/expected/single_node_update.out +++ b/src/test/regress/expected/single_node_update.out @@ -577,6 +577,45 @@ alter table t6 modify b timestamp; b | timestamp without time zone | c | timestamp without time zone | default now() +CREATE TABLE goodscheck ( +goodsid bigint, +goodscode varchar(20) DEFAULT NULL::varchar, +status integer, +isdelete integer, +introduce varchar(150) DEFAULT NULL::varchar, +createtime timestamp(0) without time zone DEFAULT NULL::timestamp without time zone, +createby varchar(20) DEFAULT NULL::varchar, +updatetime timestamp(0) without time zone DEFAULT NULL::timestamp without time zone ON UPDATE CURRENT_TIMESTAMP, +updateby varchar(20) DEFAULT NULL::varchar +); +ALTER TABLE goodscheck ADD CONSTRAINT goodscheck_pkey PRIMARY KEY (goodsid); +NOTICE: ALTER TABLE / ADD PRIMARY KEY will create implicit index "goodscheck_pkey" for table "goodscheck" +CREATE FUNCTION update_timestamp() +RETURNS trigger +LANGUAGE plpgsql +AUTHID DEFINER NOT FENCED NOT SHIPPABLE +AS $function$ +BEGIN +NEW.updateTime = now(); +RETURN NEW; +END; +$function$; +CREATE TRIGGER goodscheck_updatetime_trriger +BEFORE UPDATE OF updatetime ON goodscheck +FOR EACH ROW +EXECUTE PROCEDURE update_timestamp(); +INSERT INTO goodscheck(goodsid,goodscode,status,isdelete,introduce,createtime,createby,updatetime,updateby) +VALUES (322,'1673994937684815874',3,0,'fff','2023-07-14 10:24:51',null,'2023-08-23 10:11:30','wangjun'); +update goodscheck +set goodsId = 888, +status = 2, +introduce = 'test', +updateTime = current_timestamp, +updateBy = 'zljtest' +WHERE 1=1 +AND goodsId=322; +drop table goodscheck; +drop function update_timestamp(); -- \! @abs_bindir@/gs_dump mysql -p @portstring@ -f @abs_bindir@/dump_type.sql -F p >/dev/null 2>&1; -- create table test_feature(a int, b timestamp on update current_timestamp); -- insert into test_feature values (1); diff --git a/src/test/regress/sql/single_node_update.sql b/src/test/regress/sql/single_node_update.sql index 889d06b76..30dc1409f 100644 --- a/src/test/regress/sql/single_node_update.sql +++ b/src/test/regress/sql/single_node_update.sql @@ -231,6 +231,45 @@ alter table t6 modify b timestamp on update localtimestamp; alter table t6 modify b timestamp; \d t6 +CREATE TABLE goodscheck ( +goodsid bigint, +goodscode varchar(20) DEFAULT NULL::varchar, +status integer, +isdelete integer, +introduce varchar(150) DEFAULT NULL::varchar, +createtime timestamp(0) without time zone DEFAULT NULL::timestamp without time zone, +createby varchar(20) DEFAULT NULL::varchar, +updatetime timestamp(0) without time zone DEFAULT NULL::timestamp without time zone ON UPDATE CURRENT_TIMESTAMP, +updateby varchar(20) DEFAULT NULL::varchar +); +ALTER TABLE goodscheck ADD CONSTRAINT goodscheck_pkey PRIMARY KEY (goodsid); +CREATE FUNCTION update_timestamp() +RETURNS trigger +LANGUAGE plpgsql +AUTHID DEFINER NOT FENCED NOT SHIPPABLE +AS $function$ +BEGIN +NEW.updateTime = now(); +RETURN NEW; +END; +$function$; +CREATE TRIGGER goodscheck_updatetime_trriger +BEFORE UPDATE OF updatetime ON goodscheck +FOR EACH ROW +EXECUTE PROCEDURE update_timestamp(); +INSERT INTO goodscheck(goodsid,goodscode,status,isdelete,introduce,createtime,createby,updatetime,updateby) +VALUES (322,'1673994937684815874',3,0,'fff','2023-07-14 10:24:51',null,'2023-08-23 10:11:30','wangjun'); +update goodscheck +set goodsId = 888, +status = 2, +introduce = 'test', +updateTime = current_timestamp, +updateBy = 'zljtest' +WHERE 1=1 +AND goodsId=322; +drop table goodscheck; +drop function update_timestamp(); + -- \! @abs_bindir@/gs_dump mysql -p @portstring@ -f @abs_bindir@/dump_type.sql -F p >/dev/null 2>&1; -- create table test_feature(a int, b timestamp on update current_timestamp); From 78e11ebf8cd673cb76e0dcafda977e5d8d508285 Mon Sep 17 00:00:00 2001 From: movead Date: Thu, 14 Sep 2023 12:00:13 +0800 Subject: [PATCH 236/304] createlsnmarker for xact dispatch if enable_batch_dispatch is true --- .../storage/access/transam/parallel_recovery/dispatcher.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/gausskernel/storage/access/transam/parallel_recovery/dispatcher.cpp b/src/gausskernel/storage/access/transam/parallel_recovery/dispatcher.cpp index 91c8f84b1..40fb6dc16 100755 --- a/src/gausskernel/storage/access/transam/parallel_recovery/dispatcher.cpp +++ b/src/gausskernel/storage/access/transam/parallel_recovery/dispatcher.cpp @@ -727,6 +727,12 @@ static void DispatchToOnePageWorker(XLogReaderState *record, const RelFileNode & */ static void DispatchTxnRecord(XLogReaderState *record, List *expectedTLIs, TimestampTz recordXTime, bool imcheckpoint) { + if (g_instance.attr.attr_storage.enable_batch_dispatch) { + for (uint32 i = 0; i < g_dispatcher->pageWorkerCount; i++) { + RedoItem *item = CreateLSNMarker(record, expectedTLIs, false); + AddPageRedoItem(g_dispatcher->pageWorkers[i], item); + } + } RedoItem *trxnItem = CreateRedoItem(record, 1, ANY_WORKER, expectedTLIs, recordXTime, true); trxnItem->imcheckpoint = imcheckpoint; /* immdiate checkpoint set imcheckpoint */ AddTxnRedoItem(g_dispatcher->txnWorker, trxnItem); From e6ab8ad85bdd0d821178ca6eada7d673d1eba585 Mon Sep 17 00:00:00 2001 From: he-shaoyu Date: Mon, 11 Sep 2023 11:43:24 +0800 Subject: [PATCH 237/304] =?UTF-8?q?gs=5Fdump=E5=85=BC=E5=AE=B93.0.3?= =?UTF-8?q?=E7=89=88=E6=9C=AC=E6=95=B0=E6=8D=AE=E5=BA=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/pg_dump/pg_dump.cpp | 74 +++++++++++++++++++++++++++++-------- src/bin/pg_dump/pg_dump.h | 2 + 2 files changed, 60 insertions(+), 16 deletions(-) diff --git a/src/bin/pg_dump/pg_dump.cpp b/src/bin/pg_dump/pg_dump.cpp index f7827fccd..f43c2c237 100644 --- a/src/bin/pg_dump/pg_dump.cpp +++ b/src/bin/pg_dump/pg_dump.cpp @@ -64,6 +64,7 @@ #include "catalog/pg_trigger.h" #include "catalog/pg_type.h" #include "catalog/pg_attrdef.h" +#include "catalog/pg_partition.h" #include "libpq/libpq-fs.h" #include "libpq/libpq-int.h" #include "catalog/pgxc_node.h" @@ -5737,23 +5738,29 @@ bool IsRbObject(Archive* fout, Oid classid, Oid objid, const char* objname) /* Make sure we are in proper schema */ selectSourceSchema(fout, "pg_catalog"); - appendPQExpBuffer(query, - "SELECT pg_catalog.gs_is_recycle_obj(%u, %u, NULL)", - classid, - objid); - res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); + if (FuncExists(fout, "pg_catalog", "gs_is_recycle_obj")) { + appendPQExpBuffer(query, + "SELECT pg_catalog.gs_is_recycle_obj(%u, %u, NULL)", + classid, + objid); + res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); - colNum = PQfnumber(res, "gs_is_recycle_obj"); - recycleObject = gs_strdup(PQgetvalue(res, tupNum, colNum)); - if (strcmp(recycleObject, f) == 0) { - isRecycleObj = false; + colNum = PQfnumber(res, "gs_is_recycle_obj"); + recycleObject = gs_strdup(PQgetvalue(res, tupNum, colNum)); + if (strcmp(recycleObject, f) == 0) { + isRecycleObj = false; + } else { + isRecycleObj = true; + } + GS_FREE(recycleObject); + PQclear(res); + destroyPQExpBuffer(query); + return isRecycleObj; } else { - isRecycleObj = true; + destroyPQExpBuffer(query); + return false; } - GS_FREE(recycleObject); - PQclear(res); - destroyPQExpBuffer(query); - return isRecycleObj; + } uint32 GetVersionNum(Archive *fout) @@ -9299,7 +9306,6 @@ void getTableAttrs(Archive* fout, TableInfo* tblinfo, int numTables) selectSourceSchema(fout, tbinfo->dobj.nmspace->dobj.name); /* find all the user attributes and their types */ - /* * we must read the attribute names in attribute number order! because * we will use the attnum to index into the attnames array later. We @@ -9323,7 +9329,13 @@ void getTableAttrs(Archive* fout, TableInfo* tblinfo, int numTables) "a.attstattarget, a.attstorage, t.typstorage, " "a.attnotnull, a.atthasdef, a.attisdropped, " "a.attlen, a.attalign, a.attislocal, a.attkvtype, t.oid AS typid, " - "CASE WHEN t.typtype = 's' THEN 'set(' || (select pg_catalog.string_agg(''''||setlabel||'''', ',' order by setsortorder) from pg_catalog.pg_set group by settypid having settypid = t.oid) || ')' " + "CASE "); + if (TabExists(fout, "pg_catalog", "pg_set")) { + appendPQExpBuffer(q, + "WHEN t.typtype = 's' THEN 'set(' || (select pg_catalog.string_agg(''''||setlabel||'''', ',' order by setsortorder) from pg_catalog.pg_set group by settypid having settypid = t.oid) || ')' "); + + } + appendPQExpBuffer(q, "WHEN t.typtype = 'e' THEN 'enum(' || (select pg_catalog.string_agg(''''||enumlabel||'''', ',' order by enumsortorder) from pg_catalog.pg_enum group by enumtypid having enumtypid = t.oid) || ')' ELSE pg_catalog.format_type(t.oid,a.atttypmod) END " "AS atttypname, " "pg_catalog.array_to_string(a.attoptions, ', ') AS attoptions, " @@ -9341,6 +9353,7 @@ void getTableAttrs(Archive* fout, TableInfo* tblinfo, int numTables) "AND a.attnum > 0::pg_catalog.int2 " "ORDER BY a.attrelid, a.attnum", tbinfo->dobj.catId.oid); + appendPQExpBuffer(ce_sql, "SELECT b.column_name, a.column_key_name, b.encryption_type, " "pg_catalog.format_type(c.atttypmod, b.data_type_original_mod) AS client_encryption_original_type from " @@ -17911,6 +17924,11 @@ static bool PartkeyexprIsNull(Archive* fout, TableInfo* tbinfo, bool isSubPart) PGresult* partkeyexpr_res = NULL; int i_partkeyexpr; bool partkeyexprIsNull = true; + ArchiveHandle* AH = (ArchiveHandle*)fout; + if (!is_column_exists(AH->connection, PartitionRelationId, "partkeyexpr")) { + return true; + } + if (!isSubPart) appendPQExpBuffer(partkeyexpr, "select partkeyexpr from pg_partition where (parttype = 'r') and (parentid in (select oid from pg_class where relname = \'%s\' and " @@ -23541,3 +23559,27 @@ static bool needIgnoreSequence(TableInfo* tbinfo) } return false; } + +bool FuncExists(Archive* fout, const char* funcNamespace, const char* funcName) +{ + char query[300]; + + bool exist = false; + ArchiveHandle* AH = (ArchiveHandle*)fout; + sprintf(query, "SELECT * FROM pg_proc a LEFT JOIN pg_namespace b on a.pronamespace=b.oid WHERE a.proname='%s' and b.nspname='%s'", funcName, funcNamespace); + + exist = isExistsSQLResult(AH->connection, query); + return exist; +} + +bool TabExists(Archive* fout, const char* schemaName, const char* tabName) +{ + char query[300]; + bool exist = false; + ArchiveHandle* AH = (ArchiveHandle*)fout; + + sprintf(query, "SELECT * FROM pg_tables WHERE schemaname='%s' and tablename='%s'", schemaName, tabName); + + exist = isExistsSQLResult(AH->connection, query); + return exist; +} diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h index ac2b9f201..8b0f394cf 100644 --- a/src/bin/pg_dump/pg_dump.h +++ b/src/bin/pg_dump/pg_dump.h @@ -622,6 +622,8 @@ extern void getPublications(Archive *fout); extern void getPublicationTables(Archive *fout, TableInfo tblinfo[], int numTables); extern void getSubscriptions(Archive *fout); extern EventTriggerInfo *getEventTriggers(Archive *fout, int *numEventTriggers); +bool FuncExists(Archive* fout, const char* funcNamespace, const char* funcName); +bool TabExists(Archive* fout, const char* schemaName, const char* tabName); #ifdef GSDUMP_LLT void stopLLT(); From c63ab6dbf814dc79478aa3b0c77d7a0af1bc2fa1 Mon Sep 17 00:00:00 2001 From: weiwentao <1375910710@qq.com> Date: Thu, 14 Sep 2023 15:39:08 +0800 Subject: [PATCH 238/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dset=20collation=5Fcon?= =?UTF-8?q?nection=20=3D=20binary;=20=E8=A7=A3=E6=9E=90binary=E4=B8=BA?= =?UTF-8?q?=E5=85=B3=E9=94=AE=E5=AD=97=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/parser/gram.y | 1 + .../regress/input/charset_connection_test.source | 6 ++++++ .../regress/output/charset_connection_test.source | 15 +++++++++++++++ 3 files changed, 22 insertions(+) diff --git a/src/common/backend/parser/gram.y b/src/common/backend/parser/gram.y index 9cc50cd8f..1ecf87960 100644 --- a/src/common/backend/parser/gram.y +++ b/src/common/backend/parser/gram.y @@ -3025,6 +3025,7 @@ opt_boolean_or_string: * is the same, so we don't need to distinguish them here. */ | ColId_or_Sconst { $$ = $1; } + | BINARY { $$ = "binary";} ; /* Timezone values can be: diff --git a/src/test/regress/input/charset_connection_test.source b/src/test/regress/input/charset_connection_test.source index 9f25735f9..9b03fc480 100644 --- a/src/test/regress/input/charset_connection_test.source +++ b/src/test/regress/input/charset_connection_test.source @@ -106,6 +106,12 @@ set collation_connection = 'POSIX'; show character_set_connection; show collation_connection; +set collation_connection = "binary"; +set collation_connection = binary; +show character_set_connection; +show collation_connection; +set names utf8; + \! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1 -c "b_format_behavior_compat_options=''" >/dev/null 2>&1 select pg_sleep(1); \c diff --git a/src/test/regress/output/charset_connection_test.source b/src/test/regress/output/charset_connection_test.source index 583ea3664..eaa468f7e 100644 --- a/src/test/regress/output/charset_connection_test.source +++ b/src/test/regress/output/charset_connection_test.source @@ -363,6 +363,21 @@ show collation_connection; utf8mb4_general_ci (1 row) +set collation_connection = "binary"; +set collation_connection = binary; +show character_set_connection; + character_set_connection +-------------------------- + SQL_ASCII +(1 row) + +show collation_connection; + collation_connection +---------------------- + binary +(1 row) + +set names utf8; \! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1 -c "b_format_behavior_compat_options=''" >/dev/null 2>&1 select pg_sleep(1); pg_sleep From 4f8ca61edc267ff7bdaf55cf316c4eb0553f5a96 Mon Sep 17 00:00:00 2001 From: justbk <249396768@qq.com> Date: Thu, 14 Sep 2023 17:14:32 +0800 Subject: [PATCH 239/304] add org.opengauss jni Fully dense state adapter --- .../libpq/jdbc/client_logic_jni/Makefile | 3 +- .../org_opengauss_jdbc_ClientLogicImpl.cpp | 792 ++++++++++++++++++ .../org_opengauss_jdbc_ClientLogicImpl.h | 148 ++++ 3 files changed, 942 insertions(+), 1 deletion(-) create mode 100644 src/common/interfaces/libpq/jdbc/client_logic_jni/org_opengauss_jdbc_ClientLogicImpl.cpp create mode 100644 src/common/interfaces/libpq/jdbc/client_logic_jni/org_opengauss_jdbc_ClientLogicImpl.h diff --git a/src/common/interfaces/libpq/jdbc/client_logic_jni/Makefile b/src/common/interfaces/libpq/jdbc/client_logic_jni/Makefile index e52814bee..51452741c 100644 --- a/src/common/interfaces/libpq/jdbc/client_logic_jni/Makefile +++ b/src/common/interfaces/libpq/jdbc/client_logic_jni/Makefile @@ -28,7 +28,8 @@ driver_error.cpp \ jni_string_convertor.cpp \ jni_util.cpp \ jni_logger.cpp \ -org_postgresql_jdbc_ClientLogicImpl.cpp +org_postgresql_jdbc_ClientLogicImpl.cpp\ +org_opengauss_jdbc_ClientLogicImpl.cpp OBJS = $(CPP_SRCS:.cpp=.o) diff --git a/src/common/interfaces/libpq/jdbc/client_logic_jni/org_opengauss_jdbc_ClientLogicImpl.cpp b/src/common/interfaces/libpq/jdbc/client_logic_jni/org_opengauss_jdbc_ClientLogicImpl.cpp new file mode 100644 index 000000000..9f56b74bb --- /dev/null +++ b/src/common/interfaces/libpq/jdbc/client_logic_jni/org_opengauss_jdbc_ClientLogicImpl.cpp @@ -0,0 +1,792 @@ +/* + * Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * org_opengauss_jdbc_ClientLogicImpl.cpp + * + * IDENTIFICATION + * src/common/interfaces/libpq/jdbc/client_logic_jni/org_opengauss_jdbc_ClientLogicImpl.cpp + * + * ------------------------------------------------------------------------- + */ + +#include "client_logic_jni.h" +#include "org_opengauss_jdbc_ClientLogicImpl.h" +#ifndef ENABLE_LITE_MODE +#include +#endif +#include +#include "libpq-fe.h" +#include "jni_logger.h" +#include "jni_string_convertor.h" +#include "jni_util.h" +#include "jni_logger.h" +#include "client_logic_data_fetcher/data_fetcher_manager.h" + +#define DELETE_ARRAY(ptr) \ + if (ptr != NULL) { \ + delete[] ptr; \ + ptr = NULL; \ + } + +static bool adjust_param_valid(const char *adjusted_param_value, const char* param_value) +{ + return adjusted_param_value && param_value && + strcmp(adjusted_param_value, param_value) != 0; +} + +static bool check_pre_param_valid(JNIEnv *env, jstring statement_name_java, jobjectArray parameters_java) +{ + if (env == NULL || statement_name_java == NULL || parameters_java == NULL) { + return false; + } + return true; +} + +/* + * Placeholder for few usefull methods in JNI pluming + */ +struct JniResult { + JniResult(JNIEnv *env, int array_length) : m_env(env), m_array_length(array_length) {} + /* * + * Initializes the array to return + * @return false on any failure or true on success + */ + bool init() + { + if (m_env == NULL) { + return false; /* Should never happen */ + } + object_class = m_env->FindClass("java/lang/Object"); + if (object_class == NULL) { + return false; /* Should never happen */ + } + array = m_env->NewObjectArray(m_array_length, object_class, NULL); + if (array == NULL) { + return false; /* Should never happen */ + } + return true; + } + /* * + * Set the array to return error + * @param status + */ + void set_error_return(DriverError *status) const + { + if (status == NULL) { + return; + } + set_error(m_env, object_class, array, status->get_error_code(), + status->get_error_message() ? status->get_error_message() : ""); + } + /* * + * sets to return OK success + */ + void set_no_error_retrun() const + { + set_no_error(m_env, object_class, array); + } + /* + * Convert java string to utf8 char * using JNIStringConvertor and handles errors if any + * @param string_convertor string converter pointer + * @param java_str the java string to convert + * @param status + * @param failure_message message to put in log for failures + * @return true for success and false for failures + */ + bool convert_string(JNIStringConvertor *string_convertor, jstring java_str, DriverError *status, + const char *failure_message) const + { + if (string_convertor == NULL || java_str == NULL || status == NULL) { + return false; + } + string_convertor->convert(m_env, java_str); + if (string_convertor->c_str == NULL) { + status->set_error(JNI_SYSTEM_ERROR_CODES::STRING_CREATION_FAILED); + set_error_return(status); + JNI_LOG_ERROR("string conversion failed :%s", failure_message ? failure_message : ""); + return false; + } + return true; + } + + bool from_handle(long handle, ClientLogicJNI **client_logic, DriverError *status, const char *failure_message) const + { + if (!ClientLogicJNI::from_handle(handle, client_logic, status) || *client_logic == NULL) { + JNI_LOG_ERROR("From handle failed: %ld, on: %s", (long)handle, failure_message ? failure_message : ""); + set_error_return(status); + return false; + } + return true; + } + + JNIEnv *m_env; + int m_array_length; + jobjectArray array = NULL; + jclass object_class = NULL; +}; + +/* * + * Links the client logic object with its libpq conn to the Java PgConnection Instance + * @param env pointer to JVM + * @param jdbc_cl_impl pointer back to the Java client logic impl instance + * @return java array + * [0][0] - int status code - zero for success + * [0][1] - string status description + * [1] - long - instance handle to be re-used in future calls by the same connection + */ +JNIEXPORT jobjectArray JNICALL JDBC_FUNC_NAME(linkClientLogicImpl)(JNIEnv *env, + jobject jdbc_cl_impl, jstring database_name_java) +{ + JniResult result(env, 2); + if (!result.init()) { + return result.array; /* Should never happen */ + } + if (env == NULL || jdbc_cl_impl == NULL) { + return result.array; /* Should never happen */ + } + DriverError status(0, ""); + + // Link the client logic object + ClientLogicJNI *client_logic_jni = NULL; + client_logic_jni = new (std::nothrow) ClientLogicJNI(); + if (client_logic_jni == NULL) { + status.set_error(UNEXPECTED_ERROR); + JNI_LOG_ERROR("linkClientLogicImpl failed"); + } else { + DriverError status(0, ""); + + JNIStringConvertor database_name; + database_name.convert(env, database_name_java); + if (!client_logic_jni->link_client_logic(env, jdbc_cl_impl, database_name.c_str, &status)) { + delete client_logic_jni; + client_logic_jni = NULL; + result.set_error_return(&status); + } else { + // Successful Connection + result.set_no_error_retrun(); + jlong handle = (jlong)(intptr_t)client_logic_jni; + place_jlong_in_target_array(env, handle, 1, result.array); + } + } + return result.array; +} + +JNIEXPORT jobjectArray JNICALL JDBC_FUNC_NAME(setKmsInfoImpl)(JNIEnv *env, jobject, + jlong handle, jstring key_java, jstring value_java) +{ + JNIStringConvertor key; + JNIStringConvertor value; + + JniResult result(env, 1); + if (!result.init()) { + return result.array; /* Should never happen */ + } + DriverError status(0, ""); + ClientLogicJNI *client_logic_jni = NULL; + if (!result.from_handle((long)handle, &client_logic_jni, &status, "setKmsInfoImpl")) { + return result.array; + } + + if (!result.convert_string(&key, key_java, &status, "setKmsInfo dump kms info")) { + return result.array; + } + if (!result.convert_string(&value, value_java, &status, "setKmsInfo dump kms info")) { + return result.array; + } + + if (client_logic_jni->set_kms_info(key.c_str, value.c_str)) { + result.set_no_error_retrun(); + } else { + status.set_error(INVALID_INPUT_PARAMETER, "set kms info error"); + result.set_error_return(&status); + } + + return result.array; +} + +/* + * Runs the pre query, to replace client logic field values with binary format before sending the query to the database + * server + * @param env pointer to jvm + * @param + * @param handle pointer to ClientLogicJNI instance + * @param original_query_java the query with potentially client logic values in user format + * @return java array + * [0][0] - int status code - zero for success + * [0][1] - string status description + * [1] - String - The modified query + */ +JNIEXPORT jobjectArray JNICALL JDBC_FUNC_NAME(runQueryPreProcessImpl)(JNIEnv *env, + jobject jdbc_cl_impl, jlong handle, jstring original_query_java) +{ + JniResult result(env, 2); + if (!result.init()) { + return result.array; /* Should never happen */ + } + if (env == NULL || original_query_java == NULL) { + return result.array; /* Should never happen */ + } + DriverError status(0, ""); + ClientLogicJNI *client_logic = NULL; + if (!result.from_handle((long)handle, &client_logic, &status, "runQueryPreProcess")) { + JNI_LOG_ERROR("no handle? %s", env->GetStringUTFChars(original_query_java, NULL)); + return result.array; + } + JNIStringConvertor original_query; + + original_query.convert(env, original_query_java); + if (original_query.c_str == NULL) { + status.set_error(JNI_SYSTEM_ERROR_CODES::STRING_CREATION_FAILED); + result.set_error_return(&status); + JNI_LOG_ERROR(JDBC_FUNC_STR "runQueryPreProcessImpl error code:%d text:'%s'", + status.get_error_code(), status.get_error_message() ? status.get_error_message() : ""); + return result.array; + } + const char *original_query_dup = client_logic->get_new_query(original_query.c_str); + if (original_query_dup == NULL) { + status.set_error(JNI_SYSTEM_ERROR_CODES::STRING_CREATION_FAILED); + result.set_error_return(&status); + JNI_LOG_ERROR(JDBC_FUNC_STR "runQueryPreProcessImpl error code:%d text:'%s'", + status.get_error_code(), status.get_error_message() ? status.get_error_message() : ""); + result.set_error_return(&status); + return result.array; + } + client_logic->set_jni_env_and_cl_impl(env, jdbc_cl_impl); + if (!client_logic->run_pre_query(original_query_dup, &status)) { + JNI_LOG_ERROR( + JDBC_FUNC_STR "runQueryPreProcessImpl failed: %ld, error code: %d error: '%s'", + (long)handle, status.get_error_code(), status.get_error_message() ? status.get_error_message() : ""); + result.set_error_return(&status); + return result.array; + } + result.set_no_error_retrun(); + place_string_in_array(env, client_logic->get_pre_query_result(), 1, result.array); + client_logic->clean_stmnt(); + return result.array; +} + +/* + * Runs post process on the backend, to free the client logic state machine when a query is done + * @param env pointer to jvm + * @param + * @param handle pointer to ClientLogicJNI instance + * @param statament_name_java when issued for prepared statement contains the statement name, otherwise an empty string + * @return java array + * [0][0] - int status code - zero for success + * [0][1] - string status description + */ +JNIEXPORT jobjectArray JNICALL JDBC_FUNC_NAME(runQueryPostProcessImpl)(JNIEnv *env, + jobject jdbc_cl_impl, jlong handle) +{ + JniResult result(env, 1); + if (!result.init()) { + return result.array; /* Should never happen */ + } + DriverError status(0, ""); + ClientLogicJNI *client_logic = NULL; + if (!result.from_handle((long)handle, &client_logic, &status, "runQueryPostProcess")) { + return result.array; + } + client_logic->set_jni_env_and_cl_impl(env, jdbc_cl_impl); + if (JNI_TRUE == env->IsSameObject(jdbc_cl_impl, NULL)) { + fprintf(stderr, "Client encryption run_post_query failed jobject %p was invalid\n", jdbc_cl_impl); + } + if (!client_logic->run_post_query("", &status)) { + JNI_LOG_ERROR("run_post_query failed: %ld, error code: %d error: '%s'", (long)handle, status.get_error_code(), + status.get_error_message() ? status.get_error_message() : ""); + result.set_error_return(&status); + return result.array; + } + result.set_no_error_retrun(); + return result.array; +} + +/* + * Replace client logic field value with user input - used when receiving data in a resultset + * @param env pointer to jvm + * @param + * @param handle pointer to ClientLogicJNI instance + * @param data_to_process_java the data in binary format (hexa) + * @param data_type the oid (modid) of the original field type + * @return java array with the format below: + * [0][0] - int status code - zero for success + * [0][1] - string status description + * [1] - String - The data in user format + */ +JNIEXPORT jobjectArray JNICALL JDBC_FUNC_NAME(runClientLogicImpl)(JNIEnv *env, + jobject jdbc_cl_impl, jlong handle, jstring data_to_process_java, jint data_type) +{ + JniResult result(env, 2); + if (!result.init()) { + return result.array; /* Should never happen */ + } + + if (env == NULL || data_to_process_java == NULL) { + return result.array; + } + DriverError status(0, ""); + JNIStringConvertor data_to_process; + if (!result.convert_string(&data_to_process, data_to_process_java, &status, "runClientLogicImpl")) { + return result.array; + } + + ClientLogicJNI *client_logic = NULL; + if (!result.from_handle((long)handle, &client_logic, &status, "runClientLogicImpl")) { + return result.array; + } + client_logic->set_jni_env_and_cl_impl(env, jdbc_cl_impl); + unsigned char *proccessed_data = NULL; + size_t length_output; + if (!client_logic->deprocess_value(data_to_process.c_str, data_type, &proccessed_data, length_output, &status)) { + libpq_free(proccessed_data); + JNI_LOG_ERROR(JDBC_FUNC_STR "runClientLogicImpl failed:error code: %d error: '%s'", + status.get_error_code(), status.get_error_message() ? status.get_error_message() : ""); + result.set_error_return(&status); + return result.array; + } + result.set_no_error_retrun(); + place_ustring_in_array(env, proccessed_data, 1, result.array); + libpq_free(proccessed_data); + return result.array; +} + +/** + * check if a type is holding a record type and then need to pass its values to process_record_data + * In Java this is done once for every field in the resultset and + * then we call process_record_data only for the fields that we need to + * @param env pointer to jvm + * @param + * @param handle pointer to ClientLogicJNI instance + * @param column_name_java the name of the column in the resultset + * @param oid the oid of the field type + * @return java array with the format below: + * [0][0] - int status code - zero for success + * [0][1] - string status description + * [0...n] - array of ints with actual oids that the field contain +*/ +JNIEXPORT jobjectArray JNICALL JDBC_FUNC_NAME(getRecordIDsImpl)(JNIEnv *env, jobject, + jlong handle, jstring column_name_java, jint oid) +{ + DriverError status(0, ""); + JniResult result(env, 2); + if (!result.init()) { + return result.array; /* Should never happen */ + } + + if (env == NULL || column_name_java == NULL) { + return result.array; + } + JNIStringConvertor column_type_name; + if (!result.convert_string(&column_type_name, column_name_java, &status, "getRecordIDsImpl")) { + return result.array; + } + + ClientLogicJNI *client_logic = NULL; + if (!result.from_handle((long)handle, &client_logic, &status, "getRecordIDsImpl")) { + return result.array; + } + result.set_no_error_retrun(); + size_t number_of_oids = client_logic->get_record_data_oid_length(oid, column_type_name.c_str); + if (number_of_oids > 0) { + const int *oids = client_logic->get_record_data_oids(oid, column_type_name.c_str); + place_ints_in_target_array(env, oids, number_of_oids, 1, result.array); + } + return result.array; +} + +/** + * gets a record in client logic format and returns it in user format + * @param env pointer to jvm + * @param + * @param handle pointer to ClientLogicJNI instance + * @param data_to_process_java the data in client logic format + * @param original_oids_java the result returned from getRecordIDsImpl method for this field + * @return java array with the format below: + * [0][0] - int status code - zero for success + * [0][1] - string status description + * [1] - int 0 not client logic 1 - is client logic + * [2] - String - The data in user format +*/ +JNIEXPORT jobjectArray JNICALL JDBC_FUNC_NAME(runClientLogic4RecordImpl)(JNIEnv *env, jobject, + jlong handle, jstring data_to_process_java, jintArray original_oids_java) +{ + JniResult result(env, ARRAY_SIZE + 1); + if (!result.init()) { + return result.array; /* Should never happen */ + } + if (env == NULL || data_to_process_java == NULL || original_oids_java == NULL) { + return result.array; + } + DriverError status(0, ""); + JNIStringConvertor data_to_process; + if (!result.convert_string(&data_to_process, data_to_process_java, &status, "runClientLogic4RecordImpl")) { + return result.array; + } + ClientLogicJNI *client_logic = NULL; + if (!result.from_handle((long)handle, &client_logic, &status, "runClientLogic4RecordImpl")) { + return result.array; + } + bool is_client_logic = false; + unsigned char *proccessed_data = NULL; + size_t length_output; + size_t number_of_oids = env->GetArrayLength(original_oids_java); + if (number_of_oids > 0) { + /* Gauss wil not allow more than 250 fields with client logic */ + static const size_t MAXMIMUM_NUMBER_OF_CL_FIELDS_IN_TABLE = 250; + Assert(number_of_oids < MAXMIMUM_NUMBER_OF_CL_FIELDS_IN_TABLE); + int original_oids[MAXMIMUM_NUMBER_OF_CL_FIELDS_IN_TABLE] = {0}; + jint *oids_java = env->GetIntArrayElements(original_oids_java, 0); + for (size_t index = 0; index < number_of_oids; ++index) { + original_oids[index] = oids_java[index]; + } + if (!client_logic->process_record_data(data_to_process.c_str, original_oids, number_of_oids, + &proccessed_data, &is_client_logic, length_output, &status)) { + libpq_free(proccessed_data); + JNI_LOG_ERROR( + JDBC_FUNC_STR "runClientLogic4RecordImpl failed:error code: %d error: '%s'", + status.get_error_code(), status.get_error_message() ? status.get_error_message() : ""); + result.set_error_return(&status); + return result.array; + } + result.set_no_error_retrun(); + } + if (is_client_logic) { + place_int_in_target_array(env, 1, 1, result.array); + } else { + place_int_in_target_array(env, 0, 1, result.array); + } + if (proccessed_data != NULL) { + place_ustring_in_array(env, proccessed_data, 2, result.array); + libpq_free(proccessed_data); + } + return result.array; +} + +/* + * Run prepare statement + * @param env pointer to jvm + * @param + * @param handle pointer to ClientLogicJNI instance + * @param query_java + * @param statement_name + * @param parameter_count + * @return Object in the following format: + * [0][0][0] - error code - 0 for none + * [0][1] - error text - if none, empty string + * [1] - modified query + */ +JNIEXPORT jobjectArray JNICALL JDBC_FUNC_NAME(prepareQueryImpl)(JNIEnv *env, + jobject jdbc_cl_impl, jlong handle, jstring query_java, jstring statement_name_java, jint parameter_count) +{ + JniResult result(env, 2); + if (!result.init()) { + return result.array; + } + if (env == NULL || statement_name_java == NULL || query_java == NULL) { + return result.array; + } + DriverError status(0, ""); + ClientLogicJNI *client_logic = NULL; + if (!result.from_handle((long)handle, &client_logic, &status, "prepareQuery")) { + return result.array; + } + JNIStringConvertor original_query; + + original_query.convert(env, query_java); + if (original_query.c_str == NULL) { + status.set_error(JNI_SYSTEM_ERROR_CODES::STRING_CREATION_FAILED); + result.set_error_return(&status); + JNI_LOG_ERROR("prepareQuery failed getting the query string error code:%d text:'%s'", status.get_error_code(), + status.get_error_message() ? status.get_error_message() : ""); + return result.array; + } + JNIStringConvertor statement_name; + if (!result.convert_string(&statement_name, statement_name_java, &status, "prepareQuery")) { + return result.array; + } + client_logic->set_jni_env_and_cl_impl(env, jdbc_cl_impl); + if (!client_logic->preare_statement(original_query.c_str, statement_name.c_str, parameter_count, &status)) { + JNI_LOG_ERROR("preare_statement call failed: %ld, error code: %d error: '%s'", (long)handle, + status.get_error_code(), status.get_error_message() ? status.get_error_message() : ""); + result.set_error_return(&status); + return result.array; + } + if (client_logic->get_statement_data() == NULL) { + status.set_error(STATEMENT_DATA_EMPTY); + JNI_LOG_ERROR("preare_statement get_statement_data call failed: %ld, error code: %d error: '%s'", (long)handle, + status.get_error_code(), status.get_error_message() ? status.get_error_message() : ""); + result.set_error_return(&status); + return result.array; + } + result.set_no_error_retrun(); + place_string_in_array(env, client_logic->get_statement_data()->params.adjusted_query, 1, result.array); + return result.array; +} + +/* + * Replaces parameters in prepare statement values before sending to execution + * @param[in] env JVM environment + * @param[in] + * @param[in] handle pointer to ClientLogicJNI instance + * @param[in] statement_name_java statement name + * @param[in] parameters_java list of parameters in form of Java array of strings + * @param[in] parameter_count number of parameters + * @return Object in the following format: + * [0][0] - error code - 0 for none + * [0][1][0] - error text - if none, empty string + * [1][0 ... parameter_count - 1] - array with the parameters value, if the parameter is not being replace a NULL apears + * otherwise the replaced value + * [2][0 ... parameter_count - 1] - array with the parameters' type-oids, + * if the parameter is being replaced, otherwise 0 + */ +JNIEXPORT jobjectArray JNICALL JDBC_FUNC_NAME(replaceStatementParamsImpl)(JNIEnv *env, + jobject jdbc_cl_impl, jlong handle, jstring statement_name_java, jobjectArray parameters_java) +{ + JniResult result(env, 3); + if (!result.init()) { + return result.array; + } + if (!check_pre_param_valid(env, statement_name_java, parameters_java)) { + return result.array; + } + + DriverError status(0, ""); + JNIStringConvertor statement_name; + if (!result.convert_string(&statement_name, statement_name_java, &status, + "replaceStatementParams statement_name_java")) { + return result.array; + } + ClientLogicJNI *client_logic = NULL; + if (!result.from_handle((long)handle, &client_logic, &status, "replaceStatementParams")) { + return result.array; + } + int parameter_count = env->GetArrayLength(parameters_java); + + bool convert_failure = false; + const char **param_values = NULL; + param_values = new (std::nothrow) const char *[(size_t)parameter_count]; + if (param_values == NULL) { + status.set_error(JNI_SYSTEM_ERROR_CODES::UNEXPECTED_ERROR); + result.set_error_return(&status); + JNI_LOG_ERROR("out of memory"); + return result.array; + } + JNIStringConvertor *string_convertors = NULL; + string_convertors = new (std::nothrow) JNIStringConvertor[(size_t)parameter_count]; + if (string_convertors == NULL) { + delete[] param_values; + status.set_error(JNI_SYSTEM_ERROR_CODES::UNEXPECTED_ERROR); + result.set_error_return(&status); + JNI_LOG_ERROR("out of memory"); + return result.array; + } + for (int i = 0; i < parameter_count && !convert_failure; ++i) { + jstring value = (jstring)env->GetObjectArrayElement(parameters_java, i); + string_convertors[i].convert(env, value); + if (string_convertors[i].c_str == NULL) { + status.set_error(JNI_SYSTEM_ERROR_CODES::STRING_CREATION_FAILED); + result.set_error_return(&status); + JNI_LOG_ERROR("replaceStatementParams failed getting the parameter at index %d", i); + convert_failure = true; + } else { + param_values[i] = string_convertors[i].c_str; + } + } + if (convert_failure) { + delete[] param_values; + delete[] string_convertors; + return result.array; + } + client_logic->set_jni_env_and_cl_impl(env, jdbc_cl_impl); + if (!client_logic->replace_statement_params(statement_name.c_str, param_values, parameter_count, &status)) { + JNI_LOG_ERROR("replace_statement_params failed: %ld, error code: %d error: '%s'", (long)handle, + status.get_error_code(), status.get_error_message() ? status.get_error_message() : ""); + result.set_error_return(&status); + delete[] param_values; + delete[] string_convertors; + return result.array; + } + + // After parameters values replaces, place the new values on the target array + jobjectArray parameters_array = env->NewObjectArray(parameter_count, result.object_class, NULL); + const StatementData *stmnt_data = client_logic->get_statement_data(); + + int *parameters_types_array = NULL; + if (stmnt_data->params.adjusted_paramTypes != NULL) { + // set adjusted types array to return to JNI + parameters_types_array = new (std::nothrow) int[(size_t)parameter_count]; + if (parameters_types_array == NULL) { + delete[] param_values; + delete[] string_convertors; + status.set_error(JNI_SYSTEM_ERROR_CODES::UNEXPECTED_ERROR); + result.set_error_return(&status); + JNI_LOG_ERROR("new failed"); + return result.array; + } + } + + for (int i = 0; i < parameter_count && !convert_failure; ++i) { + /* + * rawValue in INSERT could be NULL or empty string, + * we have recgonize it in preare_statement routine and make adjusted_param_values[idx] + * to the NULL value directly, therefore ignore it here just like the normal empty value. + */ + if (adjust_param_valid(stmnt_data->params.adjusted_param_values[i], param_values[i])) { + place_string_in_array(env, stmnt_data->params.adjusted_param_values[i], i, parameters_array); + } + /* return type oid for jdbc side distinguish the enc param */ + if (parameters_types_array != NULL) { + parameters_types_array[i] = (int)stmnt_data->params.adjusted_paramTypes[i]; + } + } + + /* [1][...] arryay object */ + env->SetObjectArrayElement(result.array, 1, parameters_array); + /* [2][...] arryay object */ + place_ints_in_target_array(env, parameters_types_array, parameter_count, 2, result.array); + delete[] param_values; + delete[] string_convertors; + client_logic->clean_stmnt(); + DELETE_ARRAY(parameters_types_array); + result.set_no_error_retrun(); + return result.array; +} + +/* + * replace client logic values in error message coming from the server + * For example, when inserting a duplicate unique value, + * it will change the client logic value of \x... back to the user input + * @param env java environment + * @param + * @param handle client logic instance handle + * @param original_message_java the message received from the server and need to be converted + * @return Object in the following format: + * [0][0] - error code - 0 for none + * [0][1][0] - error text - if none, empty string + * [1] - converted message - if empty then the message has not changed + */ +JNIEXPORT jobjectArray JNICALL JDBC_FUNC_NAME(replaceErrorMessageImpl)(JNIEnv *env, + jobject jdbc_cl_impl, jlong handle, jstring original_message_java) +{ + JniResult result(env, 2); + if (!result.init()) { + return result.array; + } + if (env == NULL || original_message_java == NULL) { + return result.array; + } + DriverError status(0, ""); + // Find the client logic instance: + ClientLogicJNI *client_logic = NULL; + if (!result.from_handle((long)handle, &client_logic, &status, "replaceErrorMessage")) { + return result.array; + } + JNIStringConvertor original_message; + if (!result.convert_string(&original_message, original_message_java, &status, "replaceErrorMessage")) { + return result.array; + } + client_logic->set_jni_env_and_cl_impl(env, jdbc_cl_impl); + char *converted_message = NULL; + bool retval = client_logic->replace_message(original_message.c_str, &converted_message, &status); + if (!retval) { + // returning false due to an error + if (converted_message != NULL) { + free(converted_message); + converted_message = NULL; + } + if (status.get_error_code() != 0) { + JNI_LOG_ERROR("replaceErrorMessage failed: %ld, error code: %d error: '%s'", (long)handle, + status.get_error_code(), status.get_error_message() ? status.get_error_message() : ""); + result.set_error_return(&status); + return result.array; + } + // returning false may be due to not finding anything to replace and then the string is empty + result.set_no_error_retrun(); + place_string_in_array(env, "", 1, result.array); + return result.array; + } + + result.set_no_error_retrun(); + if (converted_message == NULL) { + /* Should not happen, but just in case */ + place_string_in_array(env, "", 1, result.array); + } else { + place_string_in_array(env, converted_message, 1, result.array); + free(converted_message); + converted_message = NULL; + } + return result.array; +} +/** + * reloads the client logic cache + * @param env java environment + * @param jdbc_cl_impl pointer back to the Java client logic impl instance + * @param handle client logic instance handle + */ +JNIEXPORT void JNICALL JDBC_FUNC_NAME(reloadCacheImpl) (JNIEnv *env, + jobject jdbc_cl_impl, jlong handle) +{ + ClientLogicJNI *client_logic = NULL; + DriverError status(0, ""); + if (!ClientLogicJNI::from_handle(handle, &client_logic, &status) || client_logic == NULL) { + JNI_LOG_DEBUG("reloadCacheImpl failed: %ld, error code: %d error: '%s'", (long)handle, status.get_error_code(), + status.get_error_message() ? status.get_error_message() : ""); + return; + } + client_logic->set_jni_env_and_cl_impl(env, jdbc_cl_impl); + client_logic->reload_cache(); +} + +/** + * reloads the client logic cache ONLY if the server timestamp is later than the client timestamp + * @param env java environment + * @param jdbc_cl_impl pointer back to the Java client logic impl instance + * @param handle client logic instance handle + */ +JNIEXPORT void JNICALL JDBC_FUNC_NAME(reloadCacheIfNeededImpl) (JNIEnv *env, + jobject jdbc_cl_impl, jlong handle){ + ClientLogicJNI *client_logic = NULL; + DriverError status(0, ""); + if (!ClientLogicJNI::from_handle(handle, &client_logic, &status) || client_logic == NULL){ + JNI_LOG_DEBUG("reloadCacheIfNeededImpl failed: %ld, error code: %d error: '%s'", + (long)handle, status.get_error_code(), + status.get_error_message() ? status.get_error_message() : ""); + return; + } + client_logic->set_jni_env_and_cl_impl(env, jdbc_cl_impl); + client_logic->reload_cache_if_needed(); +} + +/* + * Class: org_postgresql_jdbc_ClientLogicImpl + * Method: destroy + * Signature: (J)V + */ +JNIEXPORT void JNICALL JDBC_FUNC_NAME(destroy)(JNIEnv *env, jobject, jlong handle) +{ + JNI_LOG_DEBUG("About to destroy handle: %ld", (long)handle); + ClientLogicJNI *client_logic = NULL; + DriverError status(0, ""); + if (!ClientLogicJNI::from_handle(handle, &client_logic, &status) || client_logic == NULL) { + JNI_LOG_DEBUG("Destroy failed: %ld, error code: %d error: '%s'", (long)handle, status.get_error_code(), + status.get_error_message() ? status.get_error_message() : ""); + return; + } else { + delete client_logic; + client_logic = NULL; + JNI_LOG_DEBUG("Handle destroyed: %ld", (long)handle); + } + return; +} diff --git a/src/common/interfaces/libpq/jdbc/client_logic_jni/org_opengauss_jdbc_ClientLogicImpl.h b/src/common/interfaces/libpq/jdbc/client_logic_jni/org_opengauss_jdbc_ClientLogicImpl.h new file mode 100644 index 000000000..5cedefbb3 --- /dev/null +++ b/src/common/interfaces/libpq/jdbc/client_logic_jni/org_opengauss_jdbc_ClientLogicImpl.h @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * org_opengauss_jdbc_ClientLogicImpl.h + * + * IDENTIFICATION + * src/common/interfaces/libpq/jdbc/client_logic_jni/org_opengauss_jdbc_ClientLogicImpl.h + * + * ------------------------------------------------------------------------- + */ + +#ifndef _Included_org_opengauss_jdbc_ClientLogicImpl +#define _Included_org_opengauss_jdbc_ClientLogicImpl + +/* DO NOT EDIT THIS FILE - it is machine generated */ +#ifndef ENABLE_LITE_MODE +#include +#endif +#ifdef __cplusplus +extern "C" { +#endif + +#define JDBC_FUNC_REPLACE(pkg, name) Java_##pkg##jdbc_ClientLogicImpl_##name +#define JDBC_FUNC_NAME(name) JDBC_FUNC_REPLACE(org_opengauss_, name) +#define JDBC_FUNC_STR "Java_org_opengauss_jdbc_ClientLogicImpl_" + +/* + * Class: org_postgresql_jdbc_ClientLogicImpl + * Method: linkClientLogicImpl + * Signature: ()[Ljava/lang/Object; + */ +JNIEXPORT jobjectArray JNICALL JDBC_FUNC_NAME(linkClientLogicImpl) + (JNIEnv *, jobject, jstring); + +JNIEXPORT jobjectArray JNICALL JDBC_FUNC_NAME(setKmsInfoImpl)(JNIEnv *env, jobject, + jlong, jstring, jstring); + +/* + * Class: org_postgresql_jdbc_ClientLogicImpl + * Method: runQueryPreProcessImpl + * Signature: (JLjava/lang/String;)[Ljava/lang/Object; + */ +JNIEXPORT jobjectArray JNICALL JDBC_FUNC_NAME(runQueryPreProcessImpl)(JNIEnv *, jobject, jlong, + jstring); + +/* + * Class: org_postgresql_jdbc_ClientLogicImpl + * Method: runQueryPostProcessImpl + * Signature: (JLjava/lang/String;)[Ljava/lang/Object; + */ +JNIEXPORT jobjectArray JNICALL JDBC_FUNC_NAME(runQueryPostProcessImpl)(JNIEnv *, jobject, + jlong); + +/* + * Class: org_postgresql_jdbc_ClientLogicImpl + * Method: runClientLogicImpl + * Signature: (JLjava/lang/String;I)[Ljava/lang/Object; + */ +JNIEXPORT jobjectArray JNICALL JDBC_FUNC_NAME(runClientLogicImpl)(JNIEnv *, jobject, jlong, + jstring, jint); + +/* + * Class: org_postgresql_jdbc_ClientLogicImpl + * Method: getRecordIDsImpl + * Signature: (JLjava/lang/String;I)[Ljava/lang/Object; + */ +JNIEXPORT jobjectArray JNICALL JDBC_FUNC_NAME(getRecordIDsImpl)(JNIEnv *, jobject, jlong, + jstring, jint); + +/* + * Class: org_postgresql_jdbc_ClientLogicImpl + * Method: runClientLogic4RecordImpl + * Signature: (JLjava/lang/String;[I)[Ljava/lang/Object; + */ +JNIEXPORT jobjectArray JNICALL JDBC_FUNC_NAME(runClientLogic4RecordImpl)(JNIEnv *, jobject, + jlong, jstring, jintArray); + +/* + * Class: org_postgresql_jdbc_ClientLogicImpl + * Method: prepareQueryImpl + * Signature: (JLjava/lang/String;Ljava/lang/String;I)[Ljava/lang/Object; + */ +JNIEXPORT jobjectArray JNICALL JDBC_FUNC_NAME(prepareQueryImpl)(JNIEnv *, jobject, jlong, + jstring, jstring, jint); + +/* + * Class: org_postgresql_jdbc_ClientLogicImpl + * Method: replaceStatementParamsImpl + * Signature: (JLjava/lang/String;[Ljava/lang/String;)[Ljava/lang/Object; + */ +JNIEXPORT jobjectArray JNICALL JDBC_FUNC_NAME(replaceStatementParamsImpl)(JNIEnv *, jobject, + jlong, jstring, jobjectArray); + +/* + * Class: org_postgresql_jdbc_ClientLogicImpl + * Method: replaceErrorMessageImpl + * Signature: (JLjava/lang/String;)[Ljava/lang/Object; + */ +JNIEXPORT jobjectArray JNICALL JDBC_FUNC_NAME(replaceErrorMessageImpl)(JNIEnv *, jobject, + jlong, jstring); + +/* + * Class: Java_org_postgresql_jdbc_ClientLogicImpl_reloadCacheIfNeededImpl + * Method: reloadCache + * Signature: (J)V + */ +JNIEXPORT void JNICALL JDBC_FUNC_NAME(reloadCacheIfNeededImpl) (JNIEnv *, jobject, jlong); + +/* + * Class: org_postgresql_jdbc_ClientLogicImpl + * Method: reloadCache + * Signature: (J)V + */ +JNIEXPORT void JNICALL JDBC_FUNC_NAME(reloadCacheImpl) + (JNIEnv *, jobject, jlong); + +/* + * Class: Java_org_postgresql_jdbc_ClientLogicImpl_reloadCacheIfNeededImpl + * Method: reloadCache + * Signature: (J)V + */ + +JNIEXPORT void JNICALL JDBC_FUNC_NAME(reloadCacheIfNeededImpl) + (JNIEnv *, jobject, jlong); + +/* + * Class: org_postgresql_jdbc_ClientLogicImpl + * Method: destroy + * Signature: (J)V + */ +JNIEXPORT void JNICALL JDBC_FUNC_NAME(destroy)(JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif From 58b08a4fc081d07255e7218e25f13d005374a4da Mon Sep 17 00:00:00 2001 From: muyulinzhong Date: Tue, 12 Sep 2023 14:24:29 +0800 Subject: [PATCH 240/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=BC=BA=E9=99=B7?= =?UTF-8?q?=E3=80=90=E4=BD=BF=E7=94=A8--enable-profiling=E5=AF=BC=E8=87=B4?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=BA=93=E5=90=AF=E5=8A=A8=E5=A4=B1=E8=B4=A5?= =?UTF-8?q?=E3=80=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/process/main/main.cpp | 17 ++++++++++++ src/gausskernel/storage/ipc/ipc.cpp | 39 --------------------------- 2 files changed, 17 insertions(+), 39 deletions(-) diff --git a/src/gausskernel/process/main/main.cpp b/src/gausskernel/process/main/main.cpp index 838490343..305c6c8bb 100755 --- a/src/gausskernel/process/main/main.cpp +++ b/src/gausskernel/process/main/main.cpp @@ -147,6 +147,23 @@ int main(int argc, char* argv[]) progname = get_progname(argv[0]); +#ifdef PROFILE_PID_DIR +{ + char* gmon_env = NULL; + gmon_env = gs_getenv_r("GMON_OUT_PREFIX"); + if(gmon_env == NULL){ + if (gs_putenv_r("GMON_OUT_PREFIX=gmon.out") == -1) { + ereport(WARNING, + (errmsg("Failed to set ENV, cannot output gmon.out under multi-progress: EnvName=%s," + " Errno=%d, Errmessage=%s.", + "GMON_OUT_PREFIX", + errno, + gs_strerror(errno)))); + } + } +} +#endif + /* * Platform-specific startup hacks */ diff --git a/src/gausskernel/storage/ipc/ipc.cpp b/src/gausskernel/storage/ipc/ipc.cpp index b35315af5..a520c237e 100644 --- a/src/gausskernel/storage/ipc/ipc.cpp +++ b/src/gausskernel/storage/ipc/ipc.cpp @@ -280,45 +280,6 @@ void proc_exit(int code) StreamNodeGroup::syncQuit(STREAM_ERROR); StreamNodeGroup::destroy(STREAM_ERROR); -#ifdef PROFILE_PID_DIR - { - /* - * If we are profiling ourself then gprof's mcleanup() is about to - * write out a profile to ./gmon.out. Since mcleanup() always uses a - * fixed file name, each backend will overwrite earlier profiles. To - * fix that, we create a separate subdirectory for each backend - * (./gprof/pid) and 'cd' to that subdirectory before we exit() - that - * forces mcleanup() to write each profile into its own directory. We - * end up with something like: $PGDATA/gprof/8829/gmon.out - * $PGDATA/gprof/8845/gmon.out ... - * - * To avoid undesirable disk space bloat, autovacuum workers are - * discriminated against: all their gmon.out files go into the same - * subdirectory. Without this, an installation that is "just sitting - * there" nonetheless eats megabytes of disk space every few seconds. - * - * Note that we do this here instead of in an on_proc_exit() callback - * because we want to ensure that this code executes last - we don't - * want to interfere with any other on_proc_exit() callback. For the - * same reason, we do not include it in proc_exit_prepare ... so if - * you are exiting in the "wrong way" you won't drop your profile in a - * nice place. - */ - char gprofDirName[32]; - errno_t rc = EOK; - if (IsAutoVacuumWorkerProcess()) - rc = snprintf_s(gprofDirName, sizeof(gprofDirName), sizeof(gprofDirName) - 1, "gprof/avworker"); - else - rc = - snprintf_s(gprofDirName, sizeof(gprofDirName), sizeof(gprofDirName) - 1, "gprof/%lu", gs_thread_self()); - - securec_check_ss(rc, "\0", "\0"); - (void)mkdir("gprof", S_IRWXU | S_IRWXG | S_IRWXO); - (void)mkdir(gprofDirName, S_IRWXU | S_IRWXG | S_IRWXO); - (void)chdir(gprofDirName); - } -#endif - /* * Thread termination does not release any application visible process resources. * So we will take care of them explicitly. From d762f7a8ff14a0706c6711815159d9ce22474b8c Mon Sep 17 00:00:00 2001 From: zhangao_za <18829237393@163.com> Date: Thu, 14 Sep 2023 15:12:42 +0800 Subject: [PATCH 241/304] =?UTF-8?q?dss=E5=A2=9E=E5=8A=A0=E6=9C=AC=E5=9C=B0?= =?UTF-8?q?dn=E6=A0=A1=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/ddes/script/dms_contrl.sh | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/gausskernel/ddes/script/dms_contrl.sh b/src/gausskernel/ddes/script/dms_contrl.sh index 9d02573f3..5c14aadf5 100644 --- a/src/gausskernel/ddes/script/dms_contrl.sh +++ b/src/gausskernel/ddes/script/dms_contrl.sh @@ -110,6 +110,27 @@ function clear_script_log fi } +function check_local_dn_disk() +{ + test_file=${GSDB_HOME}/disk_readwrite_test + timeout 5 touch ${test_file} + if [[ $? != 0 ]] + then + log "could not wrtie on local disk, ERRNO:$?." + exit 3 + fi + + timeout 5 cat ${test_file} + if [[ $? != 0 ]] + then + log "could not read on local disk, ERRNO:$?." + rm -f ${test_file} + exit 3 + fi + + rm -f ${test_file} +} + touch_logfile() { log_file=$1 @@ -208,6 +229,7 @@ function Check() log "check ${GSDB_BIN} in ${GSDB_HOME} fail." exit 1 fi + check_local_dn_disk } # 1st step: kill database From de64649afa23c4162bfabc7bf77ea60ced3c37b9 Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Thu, 14 Sep 2023 21:16:55 +0800 Subject: [PATCH 242/304] =?UTF-8?q?build=20check=E6=80=A7=E8=83=BD?= =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/pg_ctl/pg_build.cpp | 2 +- src/bin/pg_ctl/pg_ctl.cpp | 107 ------------------ src/bin/pg_rewind/parsexlog.cpp | 67 ++++++++++- .../storage/access/transam/xlogreader.cpp | 92 +++++---------- src/include/access/xlogreader.h | 5 +- src/include/tool_common.h | 1 + 6 files changed, 96 insertions(+), 178 deletions(-) diff --git a/src/bin/pg_ctl/pg_build.cpp b/src/bin/pg_ctl/pg_build.cpp index 451a2d017..29d3a8ea9 100755 --- a/src/bin/pg_ctl/pg_build.cpp +++ b/src/bin/pg_ctl/pg_build.cpp @@ -609,7 +609,7 @@ void get_conninfo(const char* filename) } if (build_mode == CROSS_CLUSTER_FULL_BUILD || build_mode == CROSS_CLUSTER_INC_BUILD || - build_mode == CROSS_CLUSTER_STANDBY_FULL_BUILD || build_mode == BUILD_CHECK) { + build_mode == CROSS_CLUSTER_STANDBY_FULL_BUILD || ss_instance_config.dss.enable_dss) { /* For shared storage cluster */ conninfo_para = config_para_cross_cluster_build; } else { diff --git a/src/bin/pg_ctl/pg_ctl.cpp b/src/bin/pg_ctl/pg_ctl.cpp index 57bf63823..734a011ca 100755 --- a/src/bin/pg_ctl/pg_ctl.cpp +++ b/src/bin/pg_ctl/pg_ctl.cpp @@ -221,8 +221,6 @@ static char remove_member_file[MAXPGPATH]; static char change_role_file[MAXPGPATH]; static char* new_role = "passive"; static char start_minority_file[MAXPGPATH]; -static int get_instance_id(void); -static int ss_get_primary_id(void); static int ss_get_inter_node_nums(const char* interconn_url); bool ss_read_config(void); static unsigned int vote_num = 0; @@ -7338,111 +7336,6 @@ static void free_ctl() FREE_AND_RESET(ss_instance_config.dss.vgdata); } -static int get_instance_id(void) -{ - PGconn* conn = NULL; - PGresult* res = NULL; - const char* sql_string = "show ss_instance_id;"; - char* instid = NULL; - - conn = get_connectionex(); - if (PQstatus(conn) != CONNECTION_OK) { - pg_log(PG_WARNING, _("could not connect to server: %s"), PQerrorMessage(conn)); - return -1; - } - - /* Get local role from the local server. */ - res = PQexec(conn, sql_string); - if (PQresultStatus(res) != PGRES_TUPLES_OK) { - PQclear(res); - pg_log(PG_WARNING, _("could not get local role from the local server: %s"), PQerrorMessage(conn)); - close_connection(); - conn = NULL; - return -1; - } - - if (PQnfields(res) != 1 || PQntuples(res) != 1) { - int ntuples = PQntuples(res); - int nfields = PQnfields(res); - - PQclear(res); - pg_log(PG_WARNING, - _("invalid response from primary server: " - "Expected 1 tuple with 1 fields, got %d tuples with %d fields."), - ntuples, - nfields); - close_connection(); - conn = NULL; - return -1; - } - - instid = PQgetvalue(res, 0, 0); - - PQclear(res); - close_connection(); - conn = NULL; - - return atoi(instid); -} - -static int ss_get_primary_id(void) -{ - if (ss_instance_config.dss.socketpath == NULL) { - return -1; - } - - if (ss_instance_config.dss.vgname == NULL) { - return -1; - } - - int fd = -1; - int len = 0; - int err = 0; - struct stat statbuf; - char control_file_path[MAXPGPATH]; - - err = memset_s(control_file_path, MAXPGPATH, 0, MAXPGPATH); - securec_check_c(err, "\0", "\0"); - err = snprintf_s(control_file_path, MAXPGPATH, MAXPGPATH - 1, "%s/pg_control", ss_instance_config.dss.vgname); - securec_check_ss_c(err, "\0", "\0"); - - if (dss_device_init(ss_instance_config.dss.socketpath, true) != DSS_SUCCESS) { - pg_log(PG_WARNING, _("failed to init dss device\n")); - exit(1); - } - - fd = open(control_file_path, O_RDONLY | PG_BINARY, 0); - if(fd < 0) { - pg_log(PG_WARNING, _("failed to open pg_contol\n")); - close(fd); - fd = -1; - exit(1); - } - - if (stat(control_file_path, &statbuf) < 0) { - pg_log(PG_WARNING, _("failed to stat pg_contol\n")); - close(fd); - fd = -1; - exit(1); - } - - len = statbuf.st_size; - char* tmpBuffer = (char*)malloc(len + 1); - - if ((read(fd, tmpBuffer, len)) != len) { - close(fd); - fd = -1; - pg_log(PG_WARNING, _("failed to read pg_contol\n")); - exit(1); - } - - ss_reformer_ctrl_t* reformerCtrl; - - /* Calculate the offset to obtain the primary_id of the last page */ - reformerCtrl = (ss_reformer_ctrl_t*)(tmpBuffer + REFORMER_CTL_INSTANCEID * PG_CONTROL_SIZE); - return reformerCtrl->primaryInstId; -} - static int ss_get_inter_node_nums(const char* interconn_url) { errno_t rc; diff --git a/src/bin/pg_rewind/parsexlog.cpp b/src/bin/pg_rewind/parsexlog.cpp index 24b183cf1..11937ffc8 100644 --- a/src/bin/pg_rewind/parsexlog.cpp +++ b/src/bin/pg_rewind/parsexlog.cpp @@ -26,6 +26,7 @@ #include "replication/slot.h" #include "access/xlogreader.h" #include "catalog/pg_control.h" +#include "storage/file/fio_device.h" #include #define CONFIG_CASCADE_STANDBY "cascade_standby" @@ -131,6 +132,50 @@ XLogRecPtr readOneRecord(const char* datadir, XLogRecPtr ptr, TimeLineID tli) return endptr; } +int SSInitXlogDir(char*** xlogDirs) +{ + int xlogDirNum = 0; + *xlogDirs = (char**)malloc(SS_MAX_INST * sizeof(char*)); + if (*xlogDirs == NULL) { + return -1; + } + + for (int i = 0; i < SS_MAX_INST; i++) { + (*xlogDirs)[i] = (char*)malloc(MAXPGPATH * sizeof(char)); + if ((*xlogDirs)[i] == NULL) { + for (int j = 0; j < i; j++) { + free((*xlogDirs)[j]); + } + free(*xlogDirs); + return -1; + } + } + + DIR* dir = opendir(ss_instance_config.dss.vgname); + struct dirent* entry = NULL; + while (dir != NULL && (entry = readdir(dir)) != NULL) { + if (strncmp(entry->d_name, "pg_xlog", strlen("pg_xlog")) == 0) { + snprintf((*xlogDirs)[xlogDirNum], MAXPGPATH, "%s/%s", ss_instance_config.dss.vgname, entry->d_name); + xlogDirNum++; + if (xlogDirNum >= SS_MAX_INST) { + break; + } + } + } + closedir(dir); + return xlogDirNum; +} + +void FreeXlogDir(char** xlogDirs) +{ + if (ss_instance_config.dss.enable_dss && xlogDirs != NULL) { + for (int i = 0; i < SS_MAX_INST; i++) { + free(xlogDirs[i]); + } + free(xlogDirs); + } +} + BuildErrorCode findCommonCheckpoint(const char* datadir, TimeLineID tli, XLogRecPtr startrec, XLogRecPtr* lastchkptrec, TimeLineID* lastchkpttli, XLogRecPtr *lastchkptredo, uint32 term) { @@ -142,7 +187,6 @@ BuildErrorCode findCommonCheckpoint(const char* datadir, TimeLineID tli, XLogRec #endif XLogRecPtr max_lsn; char returnmsg[MAX_ERR_MSG_LENTH] = {0}; - char dssxlogdir[MAXPGPATH] = {0}; pg_crc32 maxLsnCrc = 0; XLogRecord* record = NULL; XLogRecPtr searchptr; @@ -154,17 +198,26 @@ BuildErrorCode findCommonCheckpoint(const char* datadir, TimeLineID tli, XLogRec int ret = 0; TimestampTz start_time; TimestampTz current_time; + /* maybe have some xlog dir in enable_dss mode */ + char** xlogDirs = NULL; + int xlogDirNum = 0; /* * local max lsn must be exists, or change to full build. */ if (ss_instance_config.dss.enable_dss) { - max_lsn = SSFindMaxLSN(datadir_target, returnmsg, XLOG_READER_MAX_MSGLENTH, &maxLsnCrc, ss_instance_config.dss.vgname); + xlogDirNum = SSInitXlogDir(&xlogDirs); + if (xlogDirNum <= 0) { + pg_log(PG_FATAL, "init xlog dirs failed\n"); + return BUILD_FATAL; + } + max_lsn = SSFindMaxLSN(datadir_target, returnmsg, XLOG_READER_MAX_MSGLENTH, &maxLsnCrc, xlogDirs, xlogDirNum); } else { max_lsn = FindMaxLSN(datadir_target, returnmsg, XLOG_READER_MAX_MSGLENTH, &maxLsnCrc); } if (XLogRecPtrIsInvalid(max_lsn)) { pg_fatal("find max lsn fail, errmsg:%s\n", returnmsg); + FreeXlogDir(xlogDirs); return BUILD_FATAL; } pg_log(PG_PROGRESS, "find max lsn success, %s", returnmsg); @@ -174,6 +227,7 @@ BuildErrorCode findCommonCheckpoint(const char* datadir, TimeLineID tli, XLogRec xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, &readprivate); if (xlogreader == NULL) { pg_log(PG_ERROR, "out of memory\n"); + FreeXlogDir(xlogDirs); return BUILD_ERROR; } @@ -182,7 +236,6 @@ BuildErrorCode findCommonCheckpoint(const char* datadir, TimeLineID tli, XLogRec securec_check_ss_c(ret, "\0", "\0"); get_conninfo(pg_conf_file); - searchptr = max_lsn; start_time = localGetCurrentTimestamp(); current_time = start_time; @@ -192,12 +245,13 @@ BuildErrorCode findCommonCheckpoint(const char* datadir, TimeLineID tli, XLogRec "try 300s, could not find any common checkpoint, need to do full build\n"); XLogReaderFree(xlogreader); CloseXlogFile(); + FreeXlogDir(xlogDirs); return BUILD_FATAL; } uint8 info; if (ss_instance_config.dss.enable_dss) { - record = XLogReadRecordFromAllDir(ss_instance_config.dss.vgname, xlogreader, searchptr, &errormsg); + record = XLogReadRecordFromAllDir(xlogDirs, xlogDirNum, xlogreader, searchptr, &errormsg); } else { record = XLogReadRecord(xlogreader, searchptr, &errormsg); } @@ -212,6 +266,7 @@ BuildErrorCode findCommonCheckpoint(const char* datadir, TimeLineID tli, XLogRec } XLogReaderFree(xlogreader); CloseXlogFile(); + FreeXlogDir(xlogDirs); return BUILD_FATAL; } @@ -226,7 +281,7 @@ BuildErrorCode findCommonCheckpoint(const char* datadir, TimeLineID tli, XLogRec if (xlogreader->ReadRecPtr <= startrec && XLogRecGetRmid(xlogreader) == RM_XLOG_ID && (info == XLOG_CHECKPOINT_SHUTDOWN || info == XLOG_CHECKPOINT_ONLINE)) { if (checkCommonAncestorByXlog(xlogreader->ReadRecPtr, record->xl_crc, term) == true) { - CheckPoint checkPoint; + CheckPoint checkPoint; errorno = memcpy_s(&checkPoint, sizeof(CheckPoint), XLogRecGetData(xlogreader), sizeof(CheckPoint)); securec_check_c(errorno, "", ""); *lastchkptrec = searchptr; @@ -242,6 +297,7 @@ BuildErrorCode findCommonCheckpoint(const char* datadir, TimeLineID tli, XLogRec if (increment_return_code != BUILD_SUCCESS) { XLogReaderFree(xlogreader); CloseXlogFile(); + FreeXlogDir(xlogDirs); return increment_return_code; } } @@ -252,6 +308,7 @@ BuildErrorCode findCommonCheckpoint(const char* datadir, TimeLineID tli, XLogRec XLogReaderFree(xlogreader); CloseXlogFile(); + FreeXlogDir(xlogDirs); PG_CHECKBUILD_AND_RETURN(); /* no common checkpoint between target and source, need full build */ if (XLogRecPtrIsInvalid(searchptr)) { diff --git a/src/gausskernel/storage/access/transam/xlogreader.cpp b/src/gausskernel/storage/access/transam/xlogreader.cpp index 966cd3911..8891a678b 100644 --- a/src/gausskernel/storage/access/transam/xlogreader.cpp +++ b/src/gausskernel/storage/access/transam/xlogreader.cpp @@ -1326,60 +1326,40 @@ int SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, in return XLOG_BLCKSZ; } -XLogRecord* XLogReadRecordFromAllDir(char* dirPath, XLogReaderState *xlogReader, XLogRecPtr curLsn, char** errorMsg) +XLogRecord* XLogReadRecordFromAllDir(char** xlogDirs, int xlogDirNum, XLogReaderState *xlogReader, XLogRecPtr curLsn, char** errorMsg) { - DIR* dir = opendir(dirPath); - struct dirent* entry = NULL; - char xlogDirStr[MAXPGPATH]; - errno_t rc = EOK; XLogRecord* record = NULL; - - while (dir != NULL && (entry = readdir(dir)) != NULL) { - if (strncmp(entry->d_name, "pg_xlog", strlen("pg_xlog")) == 0) { - rc = snprintf_s(xlogDirStr, MAXPGPATH, MAXPGPATH - 1, "%s/%s", dirPath, entry->d_name); - securec_check_ss_c(rc, "", ""); - record = XLogReadRecord(xlogReader, curLsn, errorMsg, true, xlogDirStr); - if (record != NULL) { - break; - } else { - CLOSE_FD(xlogreadfd); - } + for (int i = 0; i < xlogDirNum; i++) { + record = XLogReadRecord(xlogReader, curLsn, errorMsg, true, xlogDirs[i]); + if (record != NULL) { + break; + } else { + CLOSE_FD(xlogreadfd); } } - (void)closedir(dir); return record; } -void FindMaxXlogFileName(char* dirPath, char* maxXLogFileName) +void SSFindMaxXlogFileName(char* maxXLogFileName, char** xlogDirs, int xlogDirNum) { - DIR* dir = opendir(dirPath); - struct dirent* entry = NULL; - char xlogDirStr[MAXPGPATH]; errno_t rc = EOK; - - while (dir != NULL && (entry = readdir(dir)) != NULL) { - if (strncmp(entry->d_name, "pg_xlog", strlen("pg_xlog")) == 0) { - rc = snprintf_s(xlogDirStr, MAXPGPATH, MAXPGPATH - 1, "%s/%s", dirPath, entry->d_name); - securec_check_ss_c(rc, "", ""); - DIR* subDir = opendir(xlogDirStr); - struct dirent* subDirEntry = NULL; - while (subDir != NULL && (subDirEntry = readdir(subDir)) != NULL) { - if (strlen(subDirEntry->d_name) == 24 && strspn(subDirEntry->d_name, "0123456789ABCDEF") == 24 && - (strlen(maxXLogFileName) == 0 || strcmp(maxXLogFileName, subDirEntry->d_name) < 0)) { - rc = strncpy_s(maxXLogFileName, MAXPGPATH, subDirEntry->d_name, strlen(subDirEntry->d_name) + 1); - securec_check(rc, "", ""); - maxXLogFileName[strlen(subDirEntry->d_name)] = '\0'; - } + for (int i = 0; i < xlogDirNum; i++) { + DIR* subDir = opendir(xlogDirs[i]); + struct dirent* subDirEntry = NULL; + while (subDir != NULL && (subDirEntry = readdir(subDir)) != NULL) { + if (strlen(subDirEntry->d_name) == 24 && strspn(subDirEntry->d_name, "0123456789ABCDEF") == 24 && + (strlen(maxXLogFileName) == 0 || strcmp(maxXLogFileName, subDirEntry->d_name) < 0)) { + rc = strncpy_s(maxXLogFileName, MAXPGPATH, subDirEntry->d_name, strlen(subDirEntry->d_name) + 1); + securec_check(rc, "", ""); + maxXLogFileName[strlen(subDirEntry->d_name)] = '\0'; } - (void)closedir(subDir); } + (void)closedir(subDir); } - (void)closedir(dir); } -XLogRecPtr SSFindMaxLSN(char *workingPath, char *returnMsg, int msgLen, pg_crc32 *maxLsnCrc, char* dssDirStr) +XLogRecPtr SSFindMaxLSN(char *workingPath, char *returnMsg, int msgLen, pg_crc32 *maxLsnCrc, char** xlogDirs, int xlogDirNum) { - struct dirent *entry = NULL; XLogReaderState *xlogReader = NULL; XLogPageReadPrivate readPrivate = { .datadir = NULL, @@ -1395,13 +1375,10 @@ XLogRecPtr SSFindMaxLSN(char *workingPath, char *returnMsg, int msgLen, pg_crc32 bool findValidXLogFile = false; uint32 xlogReadLogid = -1; uint32 xlogReadLogSeg = -1; - char dssXlogDirStr[MAXPGPATH]; errno_t rc = EOK; - DIR* dssDir = NULL; - bool breakLoops = false; /* Ranking xlog from large to small */ - FindMaxXlogFileName(dssDirStr, maxXLogFileName); + SSFindMaxXlogFileName(maxXLogFileName, xlogDirs, xlogDirNum); if (sscanf_s(maxXLogFileName, "%08X%08X%08X", &tli, &xlogReadLogid, &xlogReadLogSeg) != 3) { rc = snprintf_s(returnMsg, XLOG_READER_MAX_MSGLENTH, XLOG_READER_MAX_MSGLENTH - 1, @@ -1432,29 +1409,18 @@ XLogRecPtr SSFindMaxLSN(char *workingPath, char *returnMsg, int msgLen, pg_crc32 /* Start to find the max lsn from a valid xlogfile */ startLsn = (xlogReadLogSeg * XLogSegSize) + ((XLogRecPtr)xlogReadLogid * XLogSegmentsPerXLogId * XLogSegSize); - while (!XLogRecPtrIsInvalid(startLsn) && !breakLoops) { - dssDir = opendir(dssDirStr); + while (!XLogRecPtrIsInvalid(startLsn) && !findValidXLogFile) { /* find the first valid record from the bigger xlogrecord. then break */ - while (dssDir != NULL && (entry = readdir(dssDir)) != NULL) { - if (strncmp(entry->d_name, "pg_xlog", strlen("pg_xlog")) == 0) { - rc = snprintf_s(dssXlogDirStr, MAXPGPATH, MAXPGPATH - 1, "%s/%s", dssDirStr, entry->d_name); -#ifndef FRONTEND - securec_check_ss(rc, "", ""); -#else - securec_check_ss_c(rc, "", ""); -#endif - curLsn = XLogFindNextRecord(xlogReader, startLsn, NULL, dssXlogDirStr); - if (XLogRecPtrIsInvalid(curLsn)) { - CLOSE_FD(xlogreadfd); - } else { - findValidXLogFile = true; - breakLoops = true; - break; - } + for (int i = 0; i < xlogDirNum; i++) { + curLsn = XLogFindNextRecord(xlogReader, startLsn, NULL, xlogDirs[i]); + if (XLogRecPtrIsInvalid(curLsn)) { + CLOSE_FD(xlogreadfd); + } else { + findValidXLogFile = true; + break; } } startLsn = startLsn - XLogSegSize; - (void)closedir(dssDir); } CLOSE_FD(xlogreadfd); @@ -1476,7 +1442,7 @@ XLogRecPtr SSFindMaxLSN(char *workingPath, char *returnMsg, int msgLen, pg_crc32 /* find the max lsn. */ while(true) { - record = XLogReadRecordFromAllDir(dssDirStr, xlogReader, curLsn, &errorMsg); + record = XLogReadRecordFromAllDir(xlogDirs, xlogDirNum, xlogReader, curLsn, &errorMsg); if (record == NULL) { break; } diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index bbd4ded83..3fe5e5dbc 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -42,7 +42,8 @@ extern void XLogReaderFree(XLogReaderState* state); /* Read the next XLog record. Returns NULL on end-of-WAL or failure */ extern struct XLogRecord* XLogReadRecord( XLogReaderState* state, XLogRecPtr recptr, char** errormsg, bool doDecode = true, char* xlog_path = NULL); -extern struct XLogRecord* XLogReadRecordFromAllDir(char* dirPath, XLogReaderState *xlogReader, XLogRecPtr curLsn, char** errorMsg); +extern struct XLogRecord* XLogReadRecordFromAllDir( + char** xlogDirs, int xlogDirNum, XLogReaderState *xlogReader, XLogRecPtr curLsn, char** errorMsg); extern bool XLogRecGetBlockTag(XLogReaderState *record, uint8 block_id, RelFileNode *rnode, ForkNumber *forknum, BlockNumber *blknum, XLogPhyBlock *pblk = NULL); @@ -57,7 +58,7 @@ extern void XLogRecGetVMPhysicalBlock(const XLogReaderState *record, uint8 block extern void XLogReaderInvalReadState(XLogReaderState* state); extern XLogRecPtr XLogFindNextRecord(XLogReaderState* state, XLogRecPtr RecPtr, XLogRecPtr *endPtr = NULL, char* xlog_path = NULL); -extern XLogRecPtr SSFindMaxLSN(char *workingPath, char *returnMsg, int msgLen, pg_crc32 *maxLsnCrc, char* dssDirStr); +extern XLogRecPtr SSFindMaxLSN(char *workingPath, char *returnMsg, int msgLen, pg_crc32 *maxLsnCrc, char** xlogDirs, int xlogDirNum); extern XLogRecPtr FindMaxLSN(char* workingpath, char* returnmsg, int msg_len, pg_crc32* maxLsnCrc, uint32 *maxLsnLen = NULL, TimeLineID *returnTli = NULL, char* xlog_path = NULL); extern XLogRecPtr FindMinLSN(char *workingPath, char *returnMsg, int msgLen, pg_crc32 *minLsnCrc); diff --git a/src/include/tool_common.h b/src/include/tool_common.h index 3f84091ce..ba96059d0 100644 --- a/src/include/tool_common.h +++ b/src/include/tool_common.h @@ -26,6 +26,7 @@ #include "storage/file/fio_device_com.h" #define MAXPGPATH 1024 +#define SS_MAX_INST 64 #define T_SS_XLOGDIR \ (g_enable_dss ? g_datadir.xlogDir : "pg_xlog") From aaa32b31028a12b934f68b8e00f963c39fb648c6 Mon Sep 17 00:00:00 2001 From: dongning12 Date: Thu, 14 Sep 2023 09:44:55 +0800 Subject: [PATCH 243/304] =?UTF-8?q?=E3=80=90=E8=B5=84=E6=BA=90=E6=B1=A0?= =?UTF-8?q?=E5=8C=96=E3=80=91mes=20worker=E7=BA=BF=E7=A8=8B=E4=BD=BF?= =?UTF-8?q?=E7=94=A8=E8=B6=85=E6=97=B6=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ddes/adapter/ss_dms_bufmgr.cpp | 56 ++++++- .../ddes/adapter/ss_dms_callback.cpp | 147 +++++++++++++----- src/gausskernel/storage/buffer/bufmgr.cpp | 20 ++- src/include/ddes/dms/ss_dms_bufmgr.h | 3 + src/include/storage/file/fio_device_com.h | 2 +- 5 files changed, 182 insertions(+), 46 deletions(-) diff --git a/src/gausskernel/ddes/adapter/ss_dms_bufmgr.cpp b/src/gausskernel/ddes/adapter/ss_dms_bufmgr.cpp index 97656b595..816195ea1 100644 --- a/src/gausskernel/ddes/adapter/ss_dms_bufmgr.cpp +++ b/src/gausskernel/ddes/adapter/ss_dms_bufmgr.cpp @@ -944,5 +944,59 @@ long SSGetBufSleepTime(int retry_times) if (retry_times < ss_buf_retry_threshold) { return 5000L * retry_times; } - return 1000L * 1000 * 20; + return SS_BUF_MAX_WAIT_TIME; } + +bool SSLWLockAcquireTimeout(LWLock* lock, LWLockMode mode) +{ + bool get_lock = false; + int wait_tickets = 2000; + int cur_tickets = 0; + + do { + get_lock = LWLockConditionalAcquire(lock, mode); + if (get_lock) { + break; + } + + pg_usleep(1000L); + cur_tickets++; + if (cur_tickets >= wait_tickets) { + break; + } + } while (true); + + if (!get_lock) { + ereport(WARNING, (errcode(MOD_DMS), (errmsg("[SS lwlock] request LWLock:%p timeout, LWLockMode:%d, timeout:2s", + lock, mode)))); + } + return get_lock; +} + +bool SSWaitIOTimeout(BufferDesc *buf) +{ + bool ret = false; + for (;;) { + uint32 buf_state; + buf_state = LockBufHdr(buf); + UnlockBufHdr(buf, buf_state); + + if (!(buf_state & BM_IO_IN_PROGRESS)) { + ret = true; + break; + } + ret = SSLWLockAcquireTimeout(buf->io_in_progress_lock, LW_SHARED); + if (ret) { + LWLockRelease(buf->io_in_progress_lock); + } + } + + if (!ret) { + BufferTag *tag = &buf->tag; + ereport(WARNING, (errmodule(MOD_DMS), (errmsg("[SS lwlock][%u/%u/%u/%d %d-%u] SSWaitIOTimeout, " + "buf_id:%d, io_in_progress_lock:%p", + tag->rnode.spcNode, tag->rnode.dbNode, tag->rnode.relNode, tag->rnode.bucketNode, + tag->forkNum, tag->blockNum, buf->buf_id, buf->io_in_progress_lock)))); + } + return ret; +} \ No newline at end of file diff --git a/src/gausskernel/ddes/adapter/ss_dms_callback.cpp b/src/gausskernel/ddes/adapter/ss_dms_callback.cpp index 4adc3cd60..21b4ccfc4 100644 --- a/src/gausskernel/ddes/adapter/ss_dms_callback.cpp +++ b/src/gausskernel/ddes/adapter/ss_dms_callback.cpp @@ -537,6 +537,7 @@ static int tryEnterLocalPage(BufferTag *tag, dms_lock_mode_t mode, dms_buf_ctrl_ LWLock *partition_lock = NULL; BufferDesc *buf_desc = NULL; RelFileNode relfilenode = tag->rnode; + bool get_lock = false; #ifdef USE_ASSERT_CHECKING if (IsSegmentPhysicalRelNode(relfilenode)) { @@ -557,7 +558,15 @@ static int tryEnterLocalPage(BufferTag *tag, dms_lock_mode_t mode, dms_buf_ctrl_ PG_TRY(); { do { - (void)LWLockAcquire(partition_lock, LW_SHARED); + get_lock = SSLWLockAcquireTimeout(partition_lock, LW_SHARED); + if (!get_lock) { + ereport(WARNING, (errmodule(MOD_DMS), (errmsg("[SS lwlock][%u/%u/%u/%d %d-%u] request LWLock timeout, " + "lock:%p", + tag->rnode.spcNode, tag->rnode.dbNode, tag->rnode.relNode, tag->rnode.bucketNode, + tag->forkNum, tag->blockNum, partition_lock)))); + ret = GS_TIMEOUT; + break; + } buf_id = BufTableLookup(tag, hash); if (buf_id < 0) { LWLockRelease(partition_lock); @@ -575,7 +584,12 @@ static int tryEnterLocalPage(BufferTag *tag, dms_lock_mode_t mode, dms_buf_ctrl_ } LWLockRelease(partition_lock); - WaitIO(buf_desc); + bool wait_success = SSWaitIOTimeout(buf_desc); + if (!wait_success) { + DmsReleaseBuffer(buf_desc->buf_id + 1, is_seg); + ret = GS_TIMEOUT; + break; + } if (!(pg_atomic_read_u32(&buf_desc->state) & BM_VALID)) { ereport(WARNING, (errmodule(MOD_DMS), @@ -600,7 +614,16 @@ static int tryEnterLocalPage(BufferTag *tag, dms_lock_mode_t mode, dms_buf_ctrl_ } LWLockMode content_mode = (mode == DMS_LOCK_SHARE) ? LW_SHARED : LW_EXCLUSIVE; - (void)LWLockAcquire(buf_desc->content_lock, content_mode); + get_lock = SSLWLockAcquireTimeout(buf_desc->content_lock, content_mode); + if (!get_lock) { + DmsReleaseBuffer(buf_desc->buf_id + 1, is_seg); + ret = GS_TIMEOUT; + ereport(WARNING, (errmodule(MOD_DMS), (errmsg("[SS lwlock][%u/%u/%u/%d %d-%u] request LWLock timeout, " + "buf_id:%d, lwlock:%p", + tag->rnode.spcNode, tag->rnode.dbNode, tag->rnode.relNode, tag->rnode.bucketNode, + tag->forkNum, tag->blockNum, buf_id, buf_desc->content_lock)))); + break; + } *buf_ctrl = GetDmsBufCtrl(buf_id); Assert(buf_id >= 0); if ((*buf_ctrl)->been_loaded == false) { @@ -686,7 +709,15 @@ static int CBInvalidatePage(void *db_handle, char pageid[DMS_PAGEID_SIZE], unsig hash = BufTableHashCode(tag); partition_lock = BufMappingPartitionLock(hash); - (void)LWLockAcquire(partition_lock, LW_SHARED); + bool get_lock = SSLWLockAcquireTimeout(partition_lock, LW_SHARED); + if (!get_lock) { + ereport(WARNING, (errmodule(MOD_DMS), (errmsg("[SS lwlock][%u/%u/%u/%d %d-%u] request LWLock timeout, " + "lwlock:%p", + tag->rnode.spcNode, tag->rnode.dbNode, tag->rnode.relNode, tag->rnode.bucketNode, + tag->forkNum, tag->blockNum, partition_lock)))); + return GS_TIMEOUT; + } + buf_id = BufTableLookup(tag, hash); if (buf_id < 0) { /* not found in shared buffer */ @@ -708,7 +739,12 @@ static int CBInvalidatePage(void *db_handle, char pageid[DMS_PAGEID_SIZE], unsig } LWLockRelease(partition_lock); - WaitIO(buf_desc); + bool wait_success = SSWaitIOTimeout(buf_desc); + if (!wait_success) { + ret = GS_TIMEOUT; + break; + } + if ((!(pg_atomic_read_u32(&buf_desc->state) & BM_VALID)) || (pg_atomic_read_u32(&buf_desc->state) & BM_IO_ERROR)) { ereport(LOG, (errmodule(MOD_DMS), @@ -721,10 +757,18 @@ static int CBInvalidatePage(void *db_handle, char pageid[DMS_PAGEID_SIZE], unsig bool can_invld_owner = (buf_desc->state & (BM_DIRTY | BM_JUST_DIRTIED | BM_PERMANENT)) > 0 ? false : true; if (!invld_owner || (invld_owner && can_invld_owner)) { - (void)LWLockAcquire(buf_desc->content_lock, LW_EXCLUSIVE); - buf_ctrl = GetDmsBufCtrl(buf_id); - buf_ctrl->lock_mode = (unsigned char)DMS_LOCK_NULL; - LWLockRelease(buf_desc->content_lock); + get_lock = SSLWLockAcquireTimeout(buf_desc->content_lock, LW_EXCLUSIVE); + if (!get_lock) { + ereport(WARNING, (errmodule(MOD_DMS), (errmsg("[SS lwlock][%u/%u/%u/%d %d-%u] request LWLock timeout, " + "buf_id:%d, lwlock:%p", + tag->rnode.spcNode, tag->rnode.dbNode, tag->rnode.relNode, tag->rnode.bucketNode, + tag->forkNum, tag->blockNum, buf_id, buf_desc->content_lock)))); + ret = GS_TIMEOUT; + } else { + buf_ctrl = GetDmsBufCtrl(buf_id); + buf_ctrl->lock_mode = (unsigned char)DMS_LOCK_NULL; + LWLockRelease(buf_desc->content_lock); + } } else { /* invalidate owner which buffer is dirty/permanent */ ereport(DEBUG1, (errmodule(MOD_DMS), errmsg("[%d/%d/%d/%d %d-%d] invalidate owner rejected, buffer is dirty/permanent, state = 0x%x", @@ -1258,13 +1302,14 @@ static int32 CBDrcBufValidate(void *db_handle) } // used for find bufferdesc in dms -static void SSGetBufferDesc(char *pageid, bool *is_valid, BufferDesc** ret_buf_desc) +static bool SSGetBufferDesc(char *pageid, bool *is_valid, BufferDesc** ret_buf_desc) { int buf_id; uint32 hash; LWLock *partition_lock = NULL; BufferTag *tag = (BufferTag *)pageid; BufferDesc *buf_desc; + bool ret = true; RelFileNode relfilenode = tag->rnode; @@ -1285,7 +1330,16 @@ static void SSGetBufferDesc(char *pageid, bool *is_valid, BufferDesc** ret_buf_d uint32 saveInterruptHoldoffCount = t_thrd.int_cxt.InterruptHoldoffCount; PG_TRY(); { - (void)LWLockAcquire(partition_lock, LW_SHARED); + bool get_lock = SSLWLockAcquireTimeout(partition_lock, LW_SHARED); + if (!get_lock) { + ereport(WARNING, (errmodule(MOD_DMS), (errmsg("[SS lwlock][%u/%u/%u/%d %d-%u] request LWLock timeout, " + "lwlock:%p", + tag->rnode.spcNode, tag->rnode.dbNode, tag->rnode.relNode, tag->rnode.bucketNode, + tag->forkNum, tag->blockNum, partition_lock)))); + ret = false; + break; + } + buf_id = BufTableLookup(tag, hash); if (buf_id >= 0) { buf_desc = GetBufferDescriptor(buf_id); @@ -1297,7 +1351,13 @@ static void SSGetBufferDesc(char *pageid, bool *is_valid, BufferDesc** ret_buf_d } LWLockRelease(partition_lock); - WaitIO(buf_desc); + bool wait_success = SSWaitIOTimeout(buf_desc); + if (!wait_success) { + SSUnPinBuffer(buf_desc); + ret = false; + break; + } + Assert(!(pg_atomic_read_u32(&buf_desc->state) & BM_IO_ERROR)); *is_valid = (pg_atomic_read_u32(&buf_desc->state) & BM_VALID) != 0; *ret_buf_desc = buf_desc; @@ -1312,6 +1372,7 @@ static void SSGetBufferDesc(char *pageid, bool *is_valid, BufferDesc** ret_buf_d ReleaseResource(); } PG_END_TRY(); + return ret; } void SSUnPinBuffer(BufferDesc* buf_desc) @@ -1330,7 +1391,13 @@ static int CBConfirmOwner(void *db_handle, char *pageid, unsigned char *lock_mod bool valid; dms_buf_ctrl_t *buf_ctrl = NULL; - SSGetBufferDesc(pageid, &valid, &buf_desc); + bool ret = SSGetBufferDesc(pageid, &valid, &buf_desc); + if (!ret) { + ereport(WARNING, (errmodule(MOD_DMS), + errmsg("[SS] CBConfirmOwner, require LWLock timeout"))); + return GS_TIMEOUT; + } + if (buf_desc == NULL) { *lock_mode = (uint8)DMS_LOCK_NULL; return GS_SUCCESS; @@ -1363,12 +1430,17 @@ static int CBConfirmConverting(void *db_handle, char *pageid, unsigned char smon BufferDesc *buf_desc = NULL; bool valid; dms_buf_ctrl_t *buf_ctrl = NULL; - bool timeout = false; *lsn = 0; *edp_map = 0; - SSGetBufferDesc(pageid, &valid, &buf_desc); + bool ret = SSGetBufferDesc(pageid, &valid, &buf_desc); + if (!ret) { + ereport(WARNING, (errmodule(MOD_DMS), + errmsg("[SS] CBConfirmConverting, require LWLock timeout"))); + return GS_TIMEOUT; + } + if (buf_desc == NULL) { *lock_mode = (uint8)DMS_LOCK_NULL; return GS_SUCCESS; @@ -1380,38 +1452,23 @@ static int CBConfirmConverting(void *db_handle, char *pageid, unsigned char smon return GS_SUCCESS; } - struct timeval begin_tv; - struct timeval now_tv; - (void)gettimeofday(&begin_tv, NULL); - long begin = GET_US(begin_tv); - long now; - - while (true) { - bool is_locked = LWLockConditionalAcquire(buf_desc->io_in_progress_lock, LW_EXCLUSIVE); - if (is_locked) { - buf_ctrl = GetDmsBufCtrl(buf_desc->buf_id); - *lock_mode = buf_ctrl->lock_mode; - LWLockRelease(buf_desc->io_in_progress_lock); - break; - } - - (void)gettimeofday(&now_tv, NULL); - now = GET_US(now_tv); - if (now - begin > REFORM_CONFIRM_TIMEOUT) { - timeout = true; - break; - } - pg_usleep(REFORM_CONFIRM_INTERVAL); /* sleep 5ms */ - } - - if (!timeout) { + bool get_lock = SSLWLockAcquireTimeout(buf_desc->io_in_progress_lock, LW_EXCLUSIVE); + if (get_lock) { + buf_ctrl = GetDmsBufCtrl(buf_desc->buf_id); + *lock_mode = buf_ctrl->lock_mode; + LWLockRelease(buf_desc->io_in_progress_lock); SSUnPinBuffer(buf_desc); return GS_SUCCESS; } if (smon_chk) { SSUnPinBuffer(buf_desc); - return GS_TIMEDOUT; + BufferTag *tag = &buf_desc->tag; + ereport(WARNING, (errmodule(MOD_DMS), (errmsg("[SS lwlock][%u/%u/%u/%d %d-%u] request LWLock timeout, " + "buf_id:%d, lwlock:%p", + tag->rnode.spcNode, tag->rnode.dbNode, tag->rnode.relNode, tag->rnode.bucketNode, + tag->forkNum, tag->blockNum, buf_desc->buf_id, buf_desc->io_in_progress_lock)))); + return GS_TIMEOUT; } // without lock @@ -1840,7 +1897,13 @@ static int CBMarkNeedFlush(void *db_handle, char *pageid) BufferDesc *buf_desc = NULL; BufferTag *tag = (BufferTag *)pageid; - SSGetBufferDesc(pageid, &valid, &buf_desc); + bool ret = SSGetBufferDesc(pageid, &valid, &buf_desc); + if (!ret) { + ereport(WARNING, (errmodule(MOD_DMS), + errmsg("[SS] CBMarkNeedFlush, require LWLock timeout"))); + return GS_TIMEOUT; + } + if (buf_desc == NULL) { ereport(WARNING, (errmodule(MOD_DMS), errmsg("[SS] CBMarkNeedFlush, buf_desc not found"))); diff --git a/src/gausskernel/storage/buffer/bufmgr.cpp b/src/gausskernel/storage/buffer/bufmgr.cpp index 63cf83f6f..67d058370 100644 --- a/src/gausskernel/storage/buffer/bufmgr.cpp +++ b/src/gausskernel/storage/buffer/bufmgr.cpp @@ -5976,7 +5976,15 @@ void LockBuffer(Buffer buffer, int mode) } dms_retry_times++; - pg_usleep(SSGetBufSleepTime(dms_retry_times)); + long sleep_time = SSGetBufSleepTime(dms_retry_times); + if (sleep_time == SS_BUF_MAX_WAIT_TIME && !SS_IN_REFORM) { + volatile BufferTag *tag = &buf->tag; + ereport(WARNING, (errmodule(MOD_DMS), (errmsg("[SS buf][%u/%u/%u/%d %d-%u] request buf timeout, " + "buf_id:%d", + tag->rnode.spcNode, tag->rnode.dbNode, tag->rnode.relNode, tag->rnode.bucketNode, + tag->forkNum, tag->blockNum, buf->buf_id)))); + } + pg_usleep(sleep_time); goto retry; } } @@ -6085,7 +6093,15 @@ bool ConditionalLockBuffer(Buffer buffer) } dms_retry_times++; - pg_usleep(SSGetBufSleepTime(dms_retry_times)); + long sleep_time = SSGetBufSleepTime(dms_retry_times); + if (sleep_time == SS_BUF_MAX_WAIT_TIME && !SS_IN_REFORM) { + volatile BufferTag *tag = &buf->tag; + ereport(WARNING, (errmodule(MOD_DMS), (errmsg("[SS buf][%u/%u/%u/%d %d-%u] request buf timeout, " + "buf_id:%d", + tag->rnode.spcNode, tag->rnode.dbNode, tag->rnode.relNode, tag->rnode.bucketNode, + tag->forkNum, tag->blockNum, buf->buf_id)))); + } + pg_usleep(sleep_time); goto retry; } } diff --git a/src/include/ddes/dms/ss_dms_bufmgr.h b/src/include/ddes/dms/ss_dms_bufmgr.h index 8807ad54b..e21f5a908 100644 --- a/src/include/ddes/dms/ss_dms_bufmgr.h +++ b/src/include/ddes/dms/ss_dms_bufmgr.h @@ -29,6 +29,7 @@ #include "access/xlogproc.h" #define GetDmsBufCtrl(id) (&t_thrd.storage_cxt.dmsBufCtl[(id)]) +#define SS_BUF_MAX_WAIT_TIME (1000L * 1000 * 20) // 20s #define DmsInitLatch(drid, _type, _oid, _idx, _parent_part, _part, _uid) \ do { \ @@ -84,4 +85,6 @@ SMGR_READ_STATUS SmgrNetPageCheckRead(Oid spcNode, Oid dbNode, Oid relNode, Fork BlockNumber blockNo, char *blockbuf); void SSUnPinBuffer(BufferDesc* buf_desc); bool SSOndemandRequestPrimaryRedo(BufferTag tag); +bool SSLWLockAcquireTimeout(LWLock* lock, LWLockMode mode); +bool SSWaitIOTimeout(BufferDesc *buf); #endif diff --git a/src/include/storage/file/fio_device_com.h b/src/include/storage/file/fio_device_com.h index 007a80ef4..47faaf5d2 100644 --- a/src/include/storage/file/fio_device_com.h +++ b/src/include/storage/file/fio_device_com.h @@ -54,6 +54,6 @@ extern uint64 XLogSegmentSize; #define GS_SUCCESS 0 #define GS_ERROR (-1) -#define GS_TIMEDOUT 1 +#define GS_TIMEOUT 1 #endif /* FIO_DEVICE_COM_H */ From 1da54bd2228537c401ec05171a5dd86746aad677 Mon Sep 17 00:00:00 2001 From: hemny Date: Fri, 15 Sep 2023 09:52:57 +0800 Subject: [PATCH 244/304] =?UTF-8?q?=E6=8E=A8=E7=82=B9CBB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/ddes/ddes_commit_id | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gausskernel/ddes/ddes_commit_id b/src/gausskernel/ddes/ddes_commit_id index 6eb39ddee..8dafe7b09 100644 --- a/src/gausskernel/ddes/ddes_commit_id +++ b/src/gausskernel/ddes/ddes_commit_id @@ -1,3 +1,3 @@ dms_commit_id=d7095d9b169a766461b00aa4bc0e19b4fc8ea657 dss_commit_id=0a759717486f91cc91b19ca6b0cf0d2e48f08fcb -cbb_commit_id=7a7e77f3dec94b6b958bea12c97b92d21dfa0bb5 \ No newline at end of file +cbb_commit_id=a8ac49e215cfe26098416b44721fe76eaca2bc1d \ No newline at end of file From 92cdd8294e6553c19b38507be05276072304b5c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BD=98=E5=85=B4=E5=BD=AC?= Date: Wed, 16 Aug 2023 18:23:11 +0800 Subject: [PATCH 245/304] =?UTF-8?q?=E6=96=B0=E5=A2=9Equery=5Fnode=5Freform?= =?UTF-8?q?=5Finfo=E8=A7=86=E5=9B=BE=EF=BC=8C=E7=94=A8=E4=BA=8E=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E8=8A=82=E7=82=B9reform=E7=9B=B8=E5=85=B3=E4=BF=A1?= =?UTF-8?q?=E6=81=AF=EF=BC=8C=E5=90=8C=E6=AD=A5=E6=9C=80=E6=96=B0og?= =?UTF-8?q?=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/catalog/builtin_funcs.ini | 4 + src/common/backend/utils/adt/pgstatfuncs.cpp | 163 ++++++++++++++++++ src/common/backend/utils/init/globals.cpp | 3 +- src/gausskernel/ddes/adapter/ss_dms.cpp | 2 +- .../ddes/adapter/ss_dms_callback.cpp | 8 + .../process/threadpool/knl_instance.cpp | 3 + .../ondemand_extreme_rto/batch_redo.cpp | 5 +- .../storage/access/transam/xlog.cpp | 3 + .../rollback-post_catalog_maindb_92_911.sql | 1 + .../rollback-post_catalog_otherdb_92_911.sql | 1 + .../upgrade-post_catalog_maindb_92_911.sql | 16 ++ .../upgrade-post_catalog_otherdb_92_911.sql | 16 ++ src/include/ddes/dms/ss_dms_recovery.h | 11 ++ 13 files changed, 233 insertions(+), 3 deletions(-) create mode 100644 src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback-post_catalog_maindb_92_911.sql create mode 100644 src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback-post_catalog_otherdb_92_911.sql create mode 100644 src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_911.sql create mode 100644 src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_911.sql diff --git a/src/common/backend/catalog/builtin_funcs.ini b/src/common/backend/catalog/builtin_funcs.ini index 7bb1b311c..9baa7b069 100755 --- a/src/common/backend/catalog/builtin_funcs.ini +++ b/src/common/backend/catalog/builtin_funcs.ini @@ -12912,3 +12912,7 @@ AddFuncGroup( "gs_hot_standby_space_info", 1, AddBuiltinFunc(_0(6218), _1("gs_hot_standby_space_info"), _2(0), _3(false), _4(true), _5(gs_hot_standby_space_info), _6(2249), _7(PG_CATALOG_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(0), _12(0), _13(0), _14(false), _15(false), _16(false), _17(false), _18('s'), _19(0), _20(0), _21(6, 28, 28, 28, 28, 28, 28), _22(6, 'o', 'o', 'o', 'o', 'o', 'o'), _23(6, "base_page_file_num", "base_page_total_size", "lsn_info_meta_file_num", "lsn_info_meta_total_size", "block_info_meta_file_num", "block_info_meta_total_size"), _24(NULL), _25("gs_hot_standby_space_info"), _26(NULL), _27(NULL), _28(NULL), _29(0), _30(false), _31(NULL), _32(false), _33(NULL), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)) ), + AddFuncGroup( + "query_node_reform_info", 1, + AddBuiltinFunc(_0(2867), _1("query_node_reform_info"), _2(3), _3(true), _4(true), _5(query_node_reform_info), _6(2249), _7(PG_CATALOG_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(64), _12(0), _13(0), _14(false), _15(false), _16(false), _17(false), _18('v'), _19(0), _20(0), _21(10, INT4OID, TEXTOID, TEXTOID, TEXTOID, BOOLOID, TEXTOID, TEXTOID, INT4OID, TEXTOID, TEXTOID), _22(10,'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o'), _23(10, "reform_node_id", "reform_type", "reform_start_time", "reform_end_time", "is_reform_success", "redo_start_time", "redo_end_time", "xlog_total_bytes", "hashmap_construct_time", "action"), _24(NULL), _25("query_node_reform_info"), _26(NULL), _27(NULL), _28(NULL), _29(0), _30(false), _31(false), _32(false), _33("query node reform information"), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)) + ), diff --git a/src/common/backend/utils/adt/pgstatfuncs.cpp b/src/common/backend/utils/adt/pgstatfuncs.cpp index eedbb0a57..c8946edbf 100644 --- a/src/common/backend/utils/adt/pgstatfuncs.cpp +++ b/src/common/backend/utils/adt/pgstatfuncs.cpp @@ -79,6 +79,7 @@ #include "nodes/makefuncs.h" #include "ddes/dms/ss_dms_bufmgr.h" #include "storage/file/fio_device.h" +#include "ddes/dms/ss_dms_recovery.h" #define UINT32_ACCESS_ONCE(var) ((uint32)(*((volatile uint32*)&(var)))) #define NUM_PG_LOCKTAG_ID 12 @@ -14900,3 +14901,165 @@ Datum gs_get_index_status(PG_FUNCTION_ARGS) } #endif + +TupleDesc create_query_node_reform_info_tupdesc() +{ + int column = 10; + TupleDesc tupdesc = CreateTemplateTupleDesc(column, false); + TupleDescInitEntry(tupdesc, (AttrNumber)1, "reform_node_id", INT4OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)2, "reform_type", TEXTOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)3, "reform_start_time", TEXTOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)4, "reform_end_time", TEXTOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)5, "is_reform_success", BOOLOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)6, "redo_start_time", TEXTOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)7, "rode_end_time", TEXTOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)8, "xlog_total_bytes", INT4OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)9, "hashmap_construct_time", TEXTOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)10, "action", TEXTOID, -1, 0); + BlessTupleDesc(tupdesc); + return tupdesc; +} + +/* +* @Description : Convert timeval to string +* @in : time +* @out : buffer +*/ +void timeval_to_string(timeval time, char* buffer, int buf_size) +{ + if (buffer == NULL || buf_size == 0 || time.tv_sec == 0) { + return; + } + time_t format_time = time.tv_sec; + struct tm *p_time = localtime(&format_time); + + char tmp_buf[32] = {0}; + strftime(tmp_buf, sizeof(tmp_buf), "%Y-%m-%d %H:%M:%S", p_time); + errno_t rc = sprintf_s(buffer, buf_size - 1, "%s.%ld ", tmp_buf, time.tv_usec / 1000); + securec_check_ss(rc, "\0", "\0"); +} + + +typedef struct { + uint64 changed_inst_list; + uint64 stable_inst_list; + uint8 iterate_idx; + ss_reform_info_t reform_info; +} reform_iterate_t; +/* + * @Description : Get reform information about special node + * @in : None + * @out : None + * @return : record + */ +Datum query_node_reform_info(PG_FUNCTION_ARGS) +{ + if (!ENABLE_DMS) { + ereport(ERROR, (errmsg("[SS] cannot query query_node_reform_info without shared storage deployment!"))); + } + + if (SS_STANDBY_MODE) { + ereport(ERROR, (errmsg("[SS] cannot query query_node_reform_info at Standby node while DMS enabled!"))); + } + + FuncCallContext *funcctx = NULL; + if (SRF_IS_FIRSTCALL()) { + funcctx = SRF_FIRSTCALL_INIT(); + MemoryContext oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); + + reform_iterate_t *iterate = (reform_iterate_t *)palloc0(sizeof(reform_iterate_t)); + ss_reform_info_t reform_info = g_instance.dms_cxt.SSReformInfo; + iterate->reform_info = reform_info; + iterate->changed_inst_list = reform_info.old_bitmap ^ reform_info.new_bitmap; + iterate->stable_inst_list = reform_info.old_bitmap & reform_info.new_bitmap; + iterate->iterate_idx = 0; + + funcctx->user_fctx = (void *)iterate; + funcctx->tuple_desc = create_query_node_reform_info_tupdesc(); + MemoryContextSwitchTo(oldcontext); + } + + funcctx = SRF_PERCALL_SETUP(); + reform_iterate_t *iterate = (reform_iterate_t *)funcctx->user_fctx; + ss_reform_info_t reform_info = iterate->reform_info; + for (uint64 i = iterate->iterate_idx; i < DMS_MAX_INSTANCE; i++) { + if (!((reform_info.old_bitmap | reform_info.new_bitmap) & (1 << i))) { + continue; + } + +#define MAX_BUF_SIZE 256 + char tmp_buf[MAX_BUF_SIZE] = {0}; + Datum values[10]; + values[0] = UInt16GetDatum(i); + if (i == (uint64)SS_MY_INST_ID) { + switch (reform_info.reform_type) { + case DMS_REFORM_TYPE_FOR_NORMAL_OPENGAUSS: + values[1] = CStringGetTextDatum("Normal reform"); + break; + case DMS_REFORM_TYPE_FOR_FAILOVER_OPENGAUSS: + values[1] = CStringGetTextDatum("Failover"); + break; + case DMS_REFORM_TYPE_FOR_SWITCHOVER_OPENGAUSS: + values[1] = CStringGetTextDatum("Switchover"); + break; + default: + values[1] = CStringGetTextDatum("NULL"); + } + + timeval_to_string(reform_info.reform_start_time, tmp_buf, MAX_BUF_SIZE); + values[2] = CStringGetTextDatum(tmp_buf); + + timeval_to_string(reform_info.reform_end_time, tmp_buf, MAX_BUF_SIZE); + values[3] = CStringGetTextDatum(tmp_buf); + values[4] = BoolGetDatum(reform_info.reform_success); + + if (reform_info.reform_type == DMS_REFORM_TYPE_FOR_FAILOVER_OPENGAUSS) { + timeval_to_string(reform_info.redo_start_time, tmp_buf, MAX_BUF_SIZE); + values[5] = CStringGetTextDatum(tmp_buf); + + timeval_to_string(reform_info.redo_end_time, tmp_buf, MAX_BUF_SIZE); + values[6] = CStringGetTextDatum(tmp_buf); + + values[7] = UInt64GetDatum(reform_info.redo_total_bytes); + + timeval_to_string(reform_info.construct_hashmap, tmp_buf, MAX_BUF_SIZE); + values[8] = CStringGetTextDatum(tmp_buf); + } else { + sprintf_s(tmp_buf, MAX_BUF_SIZE, "-"); + values[5] = CStringGetTextDatum(tmp_buf); + values[6] = CStringGetTextDatum(tmp_buf); + values[7] = UInt64GetDatum(-1); + values[8] = CStringGetTextDatum(tmp_buf); + } + } else { + values[1] = CStringGetTextDatum("-"); + sprintf_s(tmp_buf, MAX_BUF_SIZE, "-"); + values[2] = CStringGetTextDatum(tmp_buf); + values[3] = CStringGetTextDatum(tmp_buf); + values[4] = BoolGetDatum(reform_info.reform_success); + values[5] = CStringGetTextDatum(tmp_buf); + values[6] = CStringGetTextDatum(tmp_buf); + values[7] = UInt64GetDatum(-1); + values[8] = CStringGetTextDatum(tmp_buf); + } + + if (iterate->changed_inst_list & (1 << i)) { + if (reform_info.old_bitmap & (1 << i)) { + sprintf_s(tmp_buf, MAX_BUF_SIZE, "kick off"); + } else { + sprintf_s(tmp_buf, MAX_BUF_SIZE, "join in"); + } + } else if (iterate->stable_inst_list & (1 << i)) { + sprintf_s(tmp_buf, MAX_BUF_SIZE, "stable"); + } + + values[9] = CStringGetTextDatum(tmp_buf); + bool nulls[10] = {false}; + HeapTuple tuple = heap_form_tuple(funcctx->tuple_desc, values, nulls); + if (tuple != NULL) { + iterate->iterate_idx++; + SRF_RETURN_NEXT(funcctx, HeapTupleGetDatum(tuple)); + } + } + SRF_RETURN_DONE(funcctx); +} diff --git a/src/common/backend/utils/init/globals.cpp b/src/common/backend/utils/init/globals.cpp index 51a9db748..d8b32165b 100644 --- a/src/common/backend/utils/init/globals.cpp +++ b/src/common/backend/utils/init/globals.cpp @@ -75,12 +75,13 @@ bool will_shutdown = false; * NEXT | 92899 | ? | ? * ********************************************/ -const uint32 GRAND_VERSION_NUM = 92910; +const uint32 GRAND_VERSION_NUM = 92911; /******************************************** * 2.VERSION NUM FOR EACH FEATURE * Please write indescending order. ********************************************/ +const uint32 NODE_REFORM_INFO_VERSION_NUM = 92911; const uint32 GB18030_2022_VERSION_NUM = 92908; const uint32 PARAM_MARK_VERSION_NUM = 92907; const uint32 TIMESCALE_DB_VERSION_NUM = 92904; diff --git a/src/gausskernel/ddes/adapter/ss_dms.cpp b/src/gausskernel/ddes/adapter/ss_dms.cpp index b95ef62e5..3cf095297 100644 --- a/src/gausskernel/ddes/adapter/ss_dms.cpp +++ b/src/gausskernel/ddes/adapter/ss_dms.cpp @@ -357,4 +357,4 @@ unsigned int dms_get_mes_max_watting_rooms(void) int dms_send_opengauss_oldest_xmin(dms_context_t *dms_ctx, unsigned long long oldest_xmin, unsigned char dest_id) { return g_ss_dms_func.dms_send_opengauss_oldest_xmin(dms_ctx, oldest_xmin, dest_id); -} \ No newline at end of file +} diff --git a/src/gausskernel/ddes/adapter/ss_dms_callback.cpp b/src/gausskernel/ddes/adapter/ss_dms_callback.cpp index 21b4ccfc4..2a7a04b92 100644 --- a/src/gausskernel/ddes/adapter/ss_dms_callback.cpp +++ b/src/gausskernel/ddes/adapter/ss_dms_callback.cpp @@ -1771,6 +1771,7 @@ static void CBReformStartNotify(void *db_handle, dms_role_t role, unsigned char unsigned long long bitmap_nodes) { ss_reform_info_t *reform_info = &g_instance.dms_cxt.SSReformInfo; + reform_info->is_hashmap_constructed = false; reform_info->reform_type = (SSReformType)reform_type; g_instance.dms_cxt.SSClusterState = NODESTATE_NORMAL; g_instance.dms_cxt.SSRecoveryInfo.reform_ready = false; @@ -1797,6 +1798,7 @@ static void CBReformStartNotify(void *db_handle, dms_role_t role, unsigned char ereport(LOG, (errmodule(MOD_DMS), errmsg("[SS failover] failover trigger."))); } } + INSTR_TIME_SET_CURRENT(reform_info->reform_start_time); reform_info->bitmap_nodes = bitmap_nodes; reform_info->dms_role = role; @@ -1820,6 +1822,8 @@ static void CBReformStartNotify(void *db_handle, dms_role_t role, unsigned char int old_primary = SSGetPrimaryInstId(); SSReadControlFile(old_primary, true); + g_instance.dms_cxt.SSReformInfo.old_bitmap = g_instance.dms_cxt.SSReformerControl.list_stable; + ereport(LOG, (errmsg("[SS reform] old cluster node bitmap: %lld", g_instance.dms_cxt.SSReformInfo.old_bitmap))); if (SS_STANDBY_FAILOVER) { AliveFailoverCleanBackends(); @@ -1855,6 +1859,10 @@ static int CBReformDoneNotify(void *db_handle) g_instance.dms_cxt.SSRecoveryInfo.failover_ckpt_status = NOT_ACTIVE; SSReadControlFile(REFORM_CTRL_PAGE); Assert(g_instance.dms_cxt.SSRecoveryInfo.in_flushcopy == false); + g_instance.dms_cxt.SSReformInfo.new_bitmap = g_instance.dms_cxt.SSReformerControl.list_stable; + ereport(LOG, (errmsg("[SS reform] new cluster node bitmap: %lld", g_instance.dms_cxt.SSReformInfo.new_bitmap))); + INSTR_TIME_SET_CURRENT(g_instance.dms_cxt.SSReformInfo.reform_end_time); + g_instance.dms_cxt.SSReformInfo.reform_success = true; ereport(LOG, (errmodule(MOD_DMS), errmsg("[SS reform/SS switchover/SS failover] Reform success, instance:%d is running.", diff --git a/src/gausskernel/process/threadpool/knl_instance.cpp b/src/gausskernel/process/threadpool/knl_instance.cpp index 756c32229..c994bdf3a 100755 --- a/src/gausskernel/process/threadpool/knl_instance.cpp +++ b/src/gausskernel/process/threadpool/knl_instance.cpp @@ -178,6 +178,9 @@ static void knl_g_dms_init(knl_g_dms_context *dms_cxt) dms_cxt->SSReformerControl.primaryInstId = -1; dms_cxt->SSReformInfo.in_reform = false; dms_cxt->SSReformInfo.dms_role = DMS_ROLE_UNKNOW; + dms_cxt->SSReformInfo.old_bitmap = 0; + dms_cxt->SSReformInfo.new_bitmap = 0; + dms_cxt->SSReformInfo.redo_total_bytes = 0; dms_cxt->SSClusterState = NODESTATE_NORMAL; dms_cxt->SSRecoveryInfo.recovery_inst_id = INVALID_INSTANCEID; dms_cxt->SSRecoveryInfo.recovery_pause_flag = true; diff --git a/src/gausskernel/storage/access/transam/ondemand_extreme_rto/batch_redo.cpp b/src/gausskernel/storage/access/transam/ondemand_extreme_rto/batch_redo.cpp index 7d5856c80..ee4938333 100644 --- a/src/gausskernel/storage/access/transam/ondemand_extreme_rto/batch_redo.cpp +++ b/src/gausskernel/storage/access/transam/ondemand_extreme_rto/batch_redo.cpp @@ -155,7 +155,10 @@ void PRRegisterBlockChangeExtended(XLogRecParseState *recordBlockState, const Re "forknum %d blkno %u", rNode.spcNode, rNode.dbNode, rNode.relNode, forkNum, blkNo))); } - + if (!g_instance.dms_cxt.SSReformInfo.is_hashmap_constructed) { + g_instance.dms_cxt.SSReformInfo.is_hashmap_constructed = true; + INSTR_TIME_SET_CURRENT(g_instance.dms_cxt.SSReformInfo.construct_hashmap); + } if (!found) { PRInitRedoItemEntry(redoItemHashEntry); } diff --git a/src/gausskernel/storage/access/transam/xlog.cpp b/src/gausskernel/storage/access/transam/xlog.cpp index 66c6aba57..165524b0e 100755 --- a/src/gausskernel/storage/access/transam/xlog.cpp +++ b/src/gausskernel/storage/access/transam/xlog.cpp @@ -9965,6 +9965,7 @@ void StartupXLOG(void) update_stop_barrier(); #endif INSTR_TIME_SET_CURRENT(rec_startTime); + INSTR_TIME_SET_CURRENT(g_instance.dms_cxt.SSReformInfo.redo_start_time); t_thrd.xlog_cxt.RedoStartLSN = t_thrd.xlog_cxt.ReadRecPtr; g_instance.comm_cxt.predo_cxt.redoPf.redo_start_ptr = t_thrd.xlog_cxt.RedoStartLSN; knl_g_set_redo_finish_status(0); @@ -10161,8 +10162,10 @@ void StartupXLOG(void) (uint32)t_thrd.xlog_cxt.EndRecPtr))); INSTR_TIME_SET_CURRENT(rec_endTime); + INSTR_TIME_SET_CURRENT(g_instance.dms_cxt.SSReformInfo.redo_end_time); INSTR_TIME_SUBTRACT(rec_endTime, rec_startTime); redoTotalBytes = t_thrd.xlog_cxt.EndRecPtr - redoStartPtr; + g_instance.dms_cxt.SSReformInfo.redo_total_bytes = redoTotalBytes; uint64 totalTime = INSTR_TIME_GET_MICROSEC(rec_endTime); uint64 speed = 0; // MB/s if (totalTime > 0) { diff --git a/src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback-post_catalog_maindb_92_911.sql b/src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback-post_catalog_maindb_92_911.sql new file mode 100644 index 000000000..0bd7dc740 --- /dev/null +++ b/src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback-post_catalog_maindb_92_911.sql @@ -0,0 +1 @@ +DROP FUNCTION IF EXISTS pg_catalog.query_node_reform_info() CASCADE; \ No newline at end of file diff --git a/src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback-post_catalog_otherdb_92_911.sql b/src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback-post_catalog_otherdb_92_911.sql new file mode 100644 index 000000000..0bd7dc740 --- /dev/null +++ b/src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback-post_catalog_otherdb_92_911.sql @@ -0,0 +1 @@ +DROP FUNCTION IF EXISTS pg_catalog.query_node_reform_info() CASCADE; \ No newline at end of file diff --git a/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_911.sql b/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_911.sql new file mode 100644 index 000000000..a94632237 --- /dev/null +++ b/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_911.sql @@ -0,0 +1,16 @@ +DROP FUNCTION IF EXISTS pg_catalog.query_node_reform_info() CASCADE; +SET LOCAL inplace_upgrade_next_system_object_oids=IUO_PROC, 2867; +CREATE FUNCTION pg_catalog.query_node_reform_info +( + int4, + OUT node_id int4, + OUT reform_type text, + OUT reform_start_time text, + OUT reform_end_time text, + OUT is_reform_success boolean, + OUT redo_start_time text, + OUT redo_end_time text, + OUT xlog_total_bytes int4, + OUT hashmap_construct_time text +) +RETURNS record LANGUAGE INTERNAL as 'query_node_reform_info'; \ No newline at end of file diff --git a/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_911.sql b/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_911.sql new file mode 100644 index 000000000..a94632237 --- /dev/null +++ b/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_911.sql @@ -0,0 +1,16 @@ +DROP FUNCTION IF EXISTS pg_catalog.query_node_reform_info() CASCADE; +SET LOCAL inplace_upgrade_next_system_object_oids=IUO_PROC, 2867; +CREATE FUNCTION pg_catalog.query_node_reform_info +( + int4, + OUT node_id int4, + OUT reform_type text, + OUT reform_start_time text, + OUT reform_end_time text, + OUT is_reform_success boolean, + OUT redo_start_time text, + OUT redo_end_time text, + OUT xlog_total_bytes int4, + OUT hashmap_construct_time text +) +RETURNS record LANGUAGE INTERNAL as 'query_node_reform_info'; \ No newline at end of file diff --git a/src/include/ddes/dms/ss_dms_recovery.h b/src/include/ddes/dms/ss_dms_recovery.h index a763ac4f3..ba4caf5e5 100644 --- a/src/include/ddes/dms/ss_dms_recovery.h +++ b/src/include/ddes/dms/ss_dms_recovery.h @@ -64,6 +64,17 @@ typedef struct st_reform_info { dms_role_t dms_role; SSReformType reform_type; unsigned long long bitmap_nodes; + timeval reform_start_time; + timeval reform_end_time; + uint64 old_bitmap; // Save the cluster nodes bitmap before REFORM + uint64 new_bitmap; // Save the cluster nodes bitmap after REFORM + + timeval redo_start_time; + timeval redo_end_time; + timeval construct_hashmap; + uint64 redo_total_bytes; + bool reform_success; + bool is_hashmap_constructed; } ss_reform_info_t; typedef enum st_failover_ckpt_status { From d7cb7b3982264f8d3199dab5b7eb2ff45dcfe37e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BD=98=E5=85=B4=E5=BD=AC?= Date: Fri, 21 Jul 2023 11:43:10 +0800 Subject: [PATCH 246/304] =?UTF-8?q?=E6=96=B0=E5=A2=9Equery=5Fpage=5Fdistri?= =?UTF-8?q?bution=5Finfo=E7=B3=BB=E7=BB=9F=E5=87=BD=E6=95=B0,=20=E6=8E=A8?= =?UTF-8?q?=E8=BF=9BDMS=20commit=5Fid?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/catalog/builtin_funcs.ini | 4 + src/common/backend/utils/adt/pgstatfuncs.cpp | 199 +++++++++++++++++- src/common/backend/utils/init/globals.cpp | 3 +- src/gausskernel/ddes/adapter/ss_dms.cpp | 7 + .../ddes/adapter/ss_dms_callback.cpp | 11 + .../rollback-post_catalog_maindb_92_912.sql | 1 + .../rollback-post_catalog_otherdb_92_912.sql | 1 + .../upgrade-post_catalog_maindb_92_912.sql | 17 ++ .../upgrade-post_catalog_otherdb_92_912.sql | 17 ++ src/include/ddes/dms/dms_api.h | 1 - src/include/ddes/dms/ss_dms.h | 4 + src/include/ddes/dms/ss_dms_bufmgr.h | 1 + 12 files changed, 263 insertions(+), 3 deletions(-) create mode 100644 src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback-post_catalog_maindb_92_912.sql create mode 100644 src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback-post_catalog_otherdb_92_912.sql create mode 100644 src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_912.sql create mode 100644 src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_912.sql diff --git a/src/common/backend/catalog/builtin_funcs.ini b/src/common/backend/catalog/builtin_funcs.ini index 9baa7b069..aa2ffc302 100755 --- a/src/common/backend/catalog/builtin_funcs.ini +++ b/src/common/backend/catalog/builtin_funcs.ini @@ -12916,3 +12916,7 @@ AddFuncGroup( "query_node_reform_info", 1, AddBuiltinFunc(_0(2867), _1("query_node_reform_info"), _2(3), _3(true), _4(true), _5(query_node_reform_info), _6(2249), _7(PG_CATALOG_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(64), _12(0), _13(0), _14(false), _15(false), _16(false), _17(false), _18('v'), _19(0), _20(0), _21(10, INT4OID, TEXTOID, TEXTOID, TEXTOID, BOOLOID, TEXTOID, TEXTOID, INT4OID, TEXTOID, TEXTOID), _22(10,'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o'), _23(10, "reform_node_id", "reform_type", "reform_start_time", "reform_end_time", "is_reform_success", "redo_start_time", "redo_end_time", "xlog_total_bytes", "hashmap_construct_time", "action"), _24(NULL), _25("query_node_reform_info"), _26(NULL), _27(NULL), _28(NULL), _29(0), _30(false), _31(false), _32(false), _33("query node reform information"), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)) ), + AddFuncGroup( + "query_page_distribution_info", 1, + AddBuiltinFunc(_0(2866), _1("query_page_distribution_info"), _2(3), _3(true), _4(true), _5(query_page_distribution_info), _6(2249), _7(PG_CATALOG_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(64), _12(0), _13(0), _14(false), _15(false), _16(false), _17(false), _18('v'), _19(0), _20(3, TEXTOID, INT4OID, INT4OID), _21(11, TEXTOID, INT4OID, INT4OID, INT4OID, BOOLOID, BOOLOID, BOOLOID, TEXTOID, OIDOID, OIDOID, BOOLOID), _22(11, 'i', 'i', 'i', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o'), _23(11, "relname", "fork", "blockno", "instance_id", "is_master", "is_owner", "is_copy", "lock_mode", "mem_lsn", "disk_lsn", "is_dirty"), _24(NULL), _25("query_page_distribution_info"), _26(NULL), _27(NULL), _28(NULL), _29(0), _30(false), _31(NULL), _32(false), _33("statistics: query page distribution information "), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)) + ), diff --git a/src/common/backend/utils/adt/pgstatfuncs.cpp b/src/common/backend/utils/adt/pgstatfuncs.cpp index c8946edbf..d9823e007 100644 --- a/src/common/backend/utils/adt/pgstatfuncs.cpp +++ b/src/common/backend/utils/adt/pgstatfuncs.cpp @@ -62,6 +62,8 @@ #include "storage/proc.h" #include "storage/procarray.h" #include "storage/buf/buf_internals.h" +#include "storage/buf/bufmgr.h" +#include "storage/buf/bufpage.h" #include "workload/cpwlm.h" #include "workload/workload.h" #include "pgxc/pgxcnode.h" @@ -14899,7 +14901,6 @@ Datum gs_get_index_status(PG_FUNCTION_ARGS) } SRF_RETURN_DONE(funcctx); } - #endif TupleDesc create_query_node_reform_info_tupdesc() @@ -15063,3 +15064,199 @@ Datum query_node_reform_info(PG_FUNCTION_ARGS) } SRF_RETURN_DONE(funcctx); } + +void buftag_get_buf_info(BufferTag tag, stat_buf_info_t *buf_info) +{ + errno_t err = memset_s(buf_info, sizeof(stat_buf_info_t), 0, sizeof(stat_buf_info_t)); + securec_check(err, "\0", "\0"); + securec_check(err, "", ""); + uint32 hash_code = BufTableHashCode(&tag); + + LWLock *lock = BufMappingPartitionLock(hash_code); + (void)LWLockAcquire(lock, LW_SHARED); + int buf_id = BufTableLookup(&tag, hash_code); + if (buf_id >= 0) { + BufferDesc *buf_desc = GetBufferDescriptor(buf_id); + + buf_info->rec_lsn = buf_desc->extra->rec_lsn; + buf_info->lsn_on_disk = buf_desc->extra->lsn_on_disk; + buf_info->aio_in_progress = buf_desc->extra->aio_in_progress; + buf_info->dirty_queue_loc = buf_desc->extra->dirty_queue_loc; + buf_info->mem_lsn = BufferGetLSN(buf_desc); + + dms_buf_ctrl_t *buf_ctrl = GetDmsBufCtrl(buf_desc->buf_id); + buf_info->lock_mode = buf_ctrl->lock_mode; + err = memcpy_s(buf_info->data, DMS_RESID_SIZE, &tag, sizeof(BufferTag)); + securec_check(err, "", ""); + LWLockRelease(lock); + } else { + LWLockRelease(lock); + ereport(INFO, (errmsg("buffer does not exist in local buffer pool "))); + } +} + +RelFileNode relname_get_relfilenode(text* relname) +{ + RelFileNode rnode; + RangeVar* relrv = makeRangeVarFromNameList(textToQualifiedNameList(relname)); + Relation rel = relation_openrv(relrv, AccessShareLock); + if (rel == NULL) { + ereport(ERROR, (errmsg("Open relation failed!"))); + return rnode; + } + RelationOpenSmgr(rel); + rnode = rel->rd_smgr->smgr_rnode.node; + relation_close(rel, AccessShareLock); + return rnode; +} + +TupleDesc create_query_page_distribution_info_tupdesc() +{ + int column = 8; + + TupleDesc tupdesc = CreateTemplateTupleDesc(column, false); + TupleDescInitEntry(tupdesc, (AttrNumber)1, "instance_id", INT4OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)2, "is_master", BOOLOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)3, "is_owner", BOOLOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)4, "is_copy", BOOLOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)5, "lock_mode", TEXTOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)6, "mem_lsn", OIDOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)7, "disk_lsn", OIDOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)8, "is_dirty", BOOLOID, -1, 0); + BlessTupleDesc(tupdesc); + return tupdesc; +} + +int compute_copy_insts_count(uint64 bitmap) +{ + int count = 0; + for (uint8 i = 0; i < DMS_MAX_INSTANCES; i++) { + uint64 tmp = (uint64)1 << i; + if (bitmap & tmp) { + count++; + } + } + return count; +} + +/* this struct is used to control the iteration during query_page_distribution_info */ +typedef struct st_dms_iterate { + stat_drc_info_t *drc_info; + uint8 iterate_idx; +} dms_iterate_t; + +Datum query_page_distribution_info_internal(text* relname, ForkNumber fork, BlockNumber blockno, PG_FUNCTION_ARGS) +{ + if (fork >= MAX_FORKNUM) { + ereport(ERROR, (errmsg("[SS] forknumber must be less than MAX_FORKNUM(4)!"))); + } + + FuncCallContext *funcctx = NULL; + unsigned char masterId = CM_INVALID_ID8; + if (SRF_IS_FIRSTCALL()) { + RelFileNode rnode = relname_get_relfilenode(relname); + + BufferTag tag; + INIT_BUFFERTAG(tag, rnode, fork, blockno); + + stat_buf_info_t buf_info; + buftag_get_buf_info(tag, &buf_info); + char resid[DMS_PAGEID_SIZE]; + errno_t rc = memcpy_s(resid, DMS_PAGEID_SIZE, &tag, sizeof(BufferTag)); + securec_check(rc, "\0", "\0"); + + funcctx = SRF_FIRSTCALL_INIT(); + MemoryContext oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); + stat_drc_info_t *drc_info = (stat_drc_info_t*)palloc0(sizeof(stat_drc_info_t)); + InitDmsBufContext(&drc_info->dms_ctx, tag); + drc_info->claimed_owner = CM_INVALID_ID8; + drc_info->buf_info[0] = buf_info; + rc = memcpy_s(drc_info->data, DMS_PAGEID_SIZE, &tag, sizeof(BufferTag)); + securec_check(rc, "\0", "\0"); + int is_found = 0; + + int ret = get_drc_info(&is_found, drc_info); + if (ret != DMS_SUCCESS) { + ereport(ERROR, (errmsg("[SS] some errors occurred while querying DRC!"))); + } + if (!is_found) { + ereport(INFO, (errmsg("[SS] could not find a DRC entry in DRC for page (%u/%u/%u/%d/%d %d-%u)!", + rnode.spcNode, rnode.dbNode, rnode.relNode, rnode.bucketNode, rnode.opt, fork, blockno))); + } + int count = compute_copy_insts_count(drc_info->copy_insts); + + dms_iterate_t *iterate = (dms_iterate_t*)palloc0(sizeof(dms_iterate_t)); + iterate->drc_info = drc_info; + iterate->iterate_idx = 0; + count = (drc_info->claimed_owner == masterId) ? count : count + 1; + + funcctx->user_fctx = (void*)iterate; + funcctx->tuple_desc = create_query_page_distribution_info_tupdesc(); + funcctx->max_calls = (!is_found) ? 0 : (count + 1); + MemoryContextSwitchTo(oldcontext); + } + funcctx = SRF_PERCALL_SETUP(); + + if (funcctx->call_cntr < funcctx->max_calls) { + Datum values[8]; + bool nulls[8] = {false}; + bool ret_tup = false; + + dms_iterate_t *iterate = (dms_iterate_t*)funcctx->user_fctx; + for (uint8 i = iterate->iterate_idx; i < DMS_MAX_INSTANCES; i++) { + uint64 tmp = (uint64)1 << i; + if ((iterate->drc_info->copy_insts & tmp) || (iterate->drc_info->claimed_owner == i) || (iterate->drc_info->master_id == i)) { + ret_tup = true; + values[0] = UInt8GetDatum(i); // instance id + values[1] = BoolGetDatum(iterate->drc_info->master_id == i); // is master? + values[2] = BoolGetDatum(iterate->drc_info->claimed_owner == i); // is owner? + if (iterate->drc_info->copy_insts & tmp) { // is copy? + values[3] = BoolGetDatum(true); + iterate->drc_info->copy_insts = iterate->drc_info->copy_insts & ~tmp; + } else { + values[3] = BoolGetDatum(false); + } + if (iterate->drc_info->buf_info[i].lock_mode == 1) { // lock mode + values[4] = CStringGetTextDatum("Share lock"); + } else if (iterate->drc_info->buf_info[i].lock_mode == 2) { + values[4] = CStringGetTextDatum("Exclusive lock"); + } else { + values[4] = CStringGetTextDatum("No lock"); + } + values[5] = UInt64GetDatum((uint64)iterate->drc_info->buf_info[i].mem_lsn); // mem lsn + values[6] = UInt64GetDatum((uint64)iterate->drc_info->buf_info[i].lsn_on_disk); // disk lsn + if (iterate->drc_info->buf_info[i].dirty_queue_loc != PG_UINT64_MAX && // is dirty? + iterate->drc_info->buf_info[i].dirty_queue_loc != 0) { + values[7] = BoolGetDatum(true); + } else { + values[7] = BoolGetDatum(false); + } + iterate->iterate_idx = i + 1; + break; + } + } + if (ret_tup) { + HeapTuple tuple = heap_form_tuple(funcctx->tuple_desc, values, nulls); + if (tuple != NULL) { + SRF_RETURN_NEXT(funcctx, HeapTupleGetDatum(tuple)); + } + } else { + SRF_RETURN_DONE(funcctx); + } + } + SRF_RETURN_DONE(funcctx); +} + +Datum query_page_distribution_info(PG_FUNCTION_ARGS) +{ + if (!ENABLE_DMS) { + ereport(ERROR, (errmsg("[SS] cannot query query_page_distribution_info without shared storage deployment!"))); + } + if (SS_STANDBY_MODE) { + ereport(ERROR, (errmsg("[SS] cannot query query_page_distribution_info at Standby node while DMS enabled!"))); + } + text* relname = PG_GETARG_TEXT_PP(0); + ForkNumber fork = PG_GETARG_INT64(1); + BlockNumber blockno = PG_GETARG_INT64(2); + return query_page_distribution_info_internal(relname, fork, blockno, fcinfo); +} \ No newline at end of file diff --git a/src/common/backend/utils/init/globals.cpp b/src/common/backend/utils/init/globals.cpp index d8b32165b..45c289ada 100644 --- a/src/common/backend/utils/init/globals.cpp +++ b/src/common/backend/utils/init/globals.cpp @@ -75,12 +75,13 @@ bool will_shutdown = false; * NEXT | 92899 | ? | ? * ********************************************/ -const uint32 GRAND_VERSION_NUM = 92911; +const uint32 GRAND_VERSION_NUM = 92912; /******************************************** * 2.VERSION NUM FOR EACH FEATURE * Please write indescending order. ********************************************/ +const uint32 PAGE_DIST_VERSION_NUM = 92912; const uint32 NODE_REFORM_INFO_VERSION_NUM = 92911; const uint32 GB18030_2022_VERSION_NUM = 92908; const uint32 PARAM_MARK_VERSION_NUM = 92907; diff --git a/src/gausskernel/ddes/adapter/ss_dms.cpp b/src/gausskernel/ddes/adapter/ss_dms.cpp index 3cf095297..4467b0a1e 100644 --- a/src/gausskernel/ddes/adapter/ss_dms.cpp +++ b/src/gausskernel/ddes/adapter/ss_dms.cpp @@ -129,6 +129,8 @@ int ss_dms_func_init() SS_RETURN_IFERR(DMS_LOAD_SYMBOL_FUNC(dms_reform_req_opengauss_ondemand_redo_buffer)); SS_RETURN_IFERR(DMS_LOAD_SYMBOL_FUNC(dms_get_mes_max_watting_rooms)); SS_RETURN_IFERR(DMS_LOAD_SYMBOL_FUNC(dms_send_opengauss_oldest_xmin)); + SS_RETURN_IFERR(DMS_LOAD_SYMBOL_FUNC(dms_get_drc_info)); + g_ss_dms_func.inited = true; return DMS_SUCCESS; } @@ -358,3 +360,8 @@ int dms_send_opengauss_oldest_xmin(dms_context_t *dms_ctx, unsigned long long ol { return g_ss_dms_func.dms_send_opengauss_oldest_xmin(dms_ctx, oldest_xmin, dest_id); } + +int get_drc_info(int* is_found, stat_drc_info_t* drc_info) +{ + return g_ss_dms_func.dms_get_drc_info(is_found, drc_info); +} diff --git a/src/gausskernel/ddes/adapter/ss_dms_callback.cpp b/src/gausskernel/ddes/adapter/ss_dms_callback.cpp index 2a7a04b92..ee5f0ded9 100644 --- a/src/gausskernel/ddes/adapter/ss_dms_callback.cpp +++ b/src/gausskernel/ddes/adapter/ss_dms_callback.cpp @@ -47,6 +47,7 @@ #include "ddes/dms/ss_dms_bufmgr.h" #include "storage/file/fio_device.h" #include "storage/buf/bufmgr.h" +#include "storage/buf/buf_internals.h" /* * Wake up startup process to replay WAL, or to notice that @@ -2045,6 +2046,14 @@ int CBOndemandRedoPageForStandby(void *block_key, int32 *redo_status) return GS_SUCCESS;; } +void CBGetBufInfo(char* resid, stat_buf_info_t *buf_info) +{ + BufferTag tag; + errno_t err = memcpy_s(&tag, DMS_RESID_SIZE, resid, DMS_RESID_SIZE); + securec_check(err, "\0", "\0"); + buftag_get_buf_info(tag, buf_info); +} + void DmsInitCallback(dms_callback_t *callback) { // used in reform @@ -2109,4 +2118,6 @@ void DmsInitCallback(dms_callback_t *callback) callback->cache_msg = CBCacheMsg; callback->need_flush = CBMarkNeedFlush; callback->update_node_oldest_xmin = CBUpdateNodeOldestXmin; + + callback->get_buf_info = CBGetBufInfo; } diff --git a/src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback-post_catalog_maindb_92_912.sql b/src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback-post_catalog_maindb_92_912.sql new file mode 100644 index 000000000..4edfd3d62 --- /dev/null +++ b/src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback-post_catalog_maindb_92_912.sql @@ -0,0 +1 @@ +DROP FUNCTION IF EXISTS pg_catalog.query_page_distribution_info() CASCADE; \ No newline at end of file diff --git a/src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback-post_catalog_otherdb_92_912.sql b/src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback-post_catalog_otherdb_92_912.sql new file mode 100644 index 000000000..4edfd3d62 --- /dev/null +++ b/src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback-post_catalog_otherdb_92_912.sql @@ -0,0 +1 @@ +DROP FUNCTION IF EXISTS pg_catalog.query_page_distribution_info() CASCADE; \ No newline at end of file diff --git a/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_912.sql b/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_912.sql new file mode 100644 index 000000000..26384365c --- /dev/null +++ b/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_912.sql @@ -0,0 +1,17 @@ +DROP FUNCTION IF EXISTS pg_catalog.query_page_distribution_info() CASCADE; +SET LOCAL inplace_upgrade_next_system_object_oids=IUO_PROC, 2866; +CREATE FUNCTION pg_catalog.query_page_distribution_info +( + text, + int4, + int4, + OUT master_id int4, + OUT is_master boolean, + OUT is_owner boolean, + OUT is_copy boolean, + OUT lock_mode text, + OUT mem_lsn bigint, + OUT disk_lsn bigint, + OUT is_dirty boolean +) +RETURNS SETOF record LANGUAGE INTERNAL as 'query_page_distribution_info'; \ No newline at end of file diff --git a/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_912.sql b/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_912.sql new file mode 100644 index 000000000..26384365c --- /dev/null +++ b/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_912.sql @@ -0,0 +1,17 @@ +DROP FUNCTION IF EXISTS pg_catalog.query_page_distribution_info() CASCADE; +SET LOCAL inplace_upgrade_next_system_object_oids=IUO_PROC, 2866; +CREATE FUNCTION pg_catalog.query_page_distribution_info +( + text, + int4, + int4, + OUT master_id int4, + OUT is_master boolean, + OUT is_owner boolean, + OUT is_copy boolean, + OUT lock_mode text, + OUT mem_lsn bigint, + OUT disk_lsn bigint, + OUT is_dirty boolean +) +RETURNS SETOF record LANGUAGE INTERNAL as 'query_page_distribution_info'; \ No newline at end of file diff --git a/src/include/ddes/dms/dms_api.h b/src/include/ddes/dms/dms_api.h index 03bd86b2d..0ffa59af7 100644 --- a/src/include/ddes/dms/dms_api.h +++ b/src/include/ddes/dms/dms_api.h @@ -882,7 +882,6 @@ typedef struct st_dms_callback { //for shared storage backup dms_set_inst_behavior set_inst_behavior; dms_db_prepare db_prepare; - dms_get_buf_info get_buf_info; } dms_callback_t; diff --git a/src/include/ddes/dms/ss_dms.h b/src/include/ddes/dms/ss_dms.h index 77ca5148e..b30c08c10 100644 --- a/src/include/ddes/dms/ss_dms.h +++ b/src/include/ddes/dms/ss_dms.h @@ -32,6 +32,7 @@ extern "C" { #endif #define SS_LIBDMS_NAME "libdms.so" +#define CM_INVALID_ID8 0xff typedef struct st_ss_dms_func { bool inited; @@ -85,6 +86,7 @@ typedef struct st_ss_dms_func { int *redo_status); unsigned int (*dms_get_mes_max_watting_rooms)(void); int (*dms_send_opengauss_oldest_xmin)(dms_context_t *dms_ctx, unsigned long long oldest_xmin, unsigned char dest_id); + int (*dms_get_drc_info)(int* is_found, stat_drc_info_t* drc_info); } ss_dms_func_t; int ss_dms_func_init(); @@ -134,6 +136,8 @@ int dms_reform_req_opengauss_ondemand_redo_buffer(dms_context_t *dms_ctx, void * unsigned int dms_get_mes_max_watting_rooms(void); int dms_send_opengauss_oldest_xmin(dms_context_t *dms_ctx, unsigned long long oldest_xmin, unsigned char dest_id); +int get_drc_info(int* is_found, stat_drc_info_t* drc_info); + #ifdef __cplusplus } #endif diff --git a/src/include/ddes/dms/ss_dms_bufmgr.h b/src/include/ddes/dms/ss_dms_bufmgr.h index e21f5a908..5602d93cc 100644 --- a/src/include/ddes/dms/ss_dms_bufmgr.h +++ b/src/include/ddes/dms/ss_dms_bufmgr.h @@ -87,4 +87,5 @@ void SSUnPinBuffer(BufferDesc* buf_desc); bool SSOndemandRequestPrimaryRedo(BufferTag tag); bool SSLWLockAcquireTimeout(LWLock* lock, LWLockMode mode); bool SSWaitIOTimeout(BufferDesc *buf); +void buftag_get_buf_info(BufferTag tag, stat_buf_info_t *buf_info); #endif From dc42da4a450f266ed6b9b7143dfb78d04798d200 Mon Sep 17 00:00:00 2001 From: hemny Date: Thu, 14 Sep 2023 19:49:41 +0800 Subject: [PATCH 247/304] =?UTF-8?q?=E6=89=93=E5=BC=80dms=E6=97=B6=EF=BC=8C?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=BA=93=E5=8F=AA=E8=83=BD=E4=BD=BF=E7=94=A8?= =?UTF-8?q?=E6=AE=B5=E9=A1=B5=E5=BC=8F=E5=AD=98=E5=82=A8=EF=BC=8C=E5=BC=80?= =?UTF-8?q?=E6=94=BErelease=E7=89=88=E6=9C=AC=E7=9A=84enable=5Fsegment?= =?UTF-8?q?=E5=8F=82=E6=95=B0=E5=8F=AF=E9=85=8D=E7=BD=AE=EF=BC=8C=E6=8F=90?= =?UTF-8?q?=E5=8D=87=E6=98=93=E7=94=A8=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/gs_guc/cluster_guc.conf | 1 + src/bin/initdb/initdb.cpp | 4 ++++ src/common/backend/utils/misc/guc/guc_storage.cpp | 3 +-- src/common/backend/utils/misc/postgresql_single.conf.sample | 1 + 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/bin/gs_guc/cluster_guc.conf b/src/bin/gs_guc/cluster_guc.conf index b52f72f7f..c6bab6b13 100755 --- a/src/bin/gs_guc/cluster_guc.conf +++ b/src/bin/gs_guc/cluster_guc.conf @@ -727,6 +727,7 @@ logical_sender_timeout|int|0,2147483647|ms|NULL| var_eq_const_selectivity|bool|0,0|NULL|NULL| enable_save_confirmed_lsn|bool|0,0|NULL|NULL| ss_enable_dss|bool|0,0|NULL|NULL| +enable_segment|bool|0,0|NULL|NULL| ss_dss_vg_name|string|0,0|NULL|NULL| ss_dss_conn_path|string|0,0|NULL|NULL| ss_enable_dms|bool|0,0|NULL|NULL| diff --git a/src/bin/initdb/initdb.cpp b/src/bin/initdb/initdb.cpp index 75d5e6bcf..676432793 100644 --- a/src/bin/initdb/initdb.cpp +++ b/src/bin/initdb/initdb.cpp @@ -1593,6 +1593,10 @@ static void setup_config(void) securec_check_c(nRet, "\0", "\0"); conflines = replace_token(conflines, "#ss_enable_dss = off", repltok); + nRet = strcpy_s(repltok, sizeof(repltok), "enable_segment = on"); + securec_check_c(nRet, "\0", "\0"); + conflines = replace_token(conflines, "#enable_segment = off", repltok); + conflines = ss_addnodeparmater(conflines); } diff --git a/src/common/backend/utils/misc/guc/guc_storage.cpp b/src/common/backend/utils/misc/guc/guc_storage.cpp index f8f7b9ee1..e00f1d1ab 100755 --- a/src/common/backend/utils/misc/guc/guc_storage.cpp +++ b/src/common/backend/utils/misc/guc/guc_storage.cpp @@ -956,7 +956,6 @@ static void InitStorageConfigureNamesBool() NULL, NULL}, -#ifdef USE_ASSERT_CHECKING {{"enable_segment", PGC_SIGHUP, NODE_ALL, @@ -968,7 +967,7 @@ static void InitStorageConfigureNamesBool() NULL, NULL, NULL}, -#endif + {{"enable_gtm_free", PGC_POSTMASTER, NODE_DISTRIBUTE, diff --git a/src/common/backend/utils/misc/postgresql_single.conf.sample b/src/common/backend/utils/misc/postgresql_single.conf.sample index 3b1eeb20a..3e78fabea 100644 --- a/src/common/backend/utils/misc/postgresql_single.conf.sample +++ b/src/common/backend/utils/misc/postgresql_single.conf.sample @@ -852,3 +852,4 @@ job_queue_processes = 10 # Number of concurrent jobs, optional: [0..1000] #ss_enable_ondemand_recovery = off #ss_ondemand_recovery_mem_size = 4GB # min: 1GB, max: 100GB #enable_ss_dorado = off +#enable_segment = off From 562e642c47b10430a4cdeecc8d685e1465200cc0 Mon Sep 17 00:00:00 2001 From: nnuanyang Date: Wed, 6 Sep 2023 00:05:08 -0700 Subject: [PATCH 248/304] show warnings number bug --- src/common/backend/utils/error/elog.cpp | 84 +------------------ src/common/pl/plpgsql/src/pl_exec.cpp | 4 +- src/include/utils/elog.h | 1 - src/test/regress/expected/mysql_condition.out | 4 +- src/test/regress/expected/mysql_resignal.out | 9 +- 5 files changed, 11 insertions(+), 91 deletions(-) diff --git a/src/common/backend/utils/error/elog.cpp b/src/common/backend/utils/error/elog.cpp index 08f510213..237554c10 100644 --- a/src/common/backend/utils/error/elog.cpp +++ b/src/common/backend/utils/error/elog.cpp @@ -5900,85 +5900,6 @@ void copyErrorDataArea(ErrorDataArea *from, ErrorDataArea *to) } MemoryContextSwitchTo(oldcontext); } -bool strcompare(char* str1, char* str2) -{ - if(str1 != NULL) { - if (str2 != NULL) { - if (strcmp(str1, str2) != 0) { - return false; - } - } else { - return true; - } - } else { - if (str2 != NULL) { - return false; - } - } - return true; -} -bool compareErrorData(DolphinErrorData* err1, ErrorData* err2) -{ - if (err1->elevel != errorLevelToDolphin(err2->elevel)) - return false; - int errcode = MAKE_SQLSTATE(err1->errorcode[0], - err1->errorcode[1], - err1->errorcode[2], - err1->errorcode[3], - err1->errorcode[4]); - if((errcode == err2->sqlerrcode) && strcompare(err1->class_origin,err2->class_origin) - && strcompare(err1->subclass_origin,err2->subclass_origin) && strcompare(err1->constraint_catalog,err2->cons_catalog) - && strcompare(err1->constraint_schema,err2->cons_schema) && strcompare(err1->constraint_name,err2->cons_name) - && strcompare(err1->catalog_name,err2->catalog_name) && strcompare(err1->schema_name,err2->schema_name) - && strcompare(err1->table_name,err2->table_name) && strcompare(err1->column_name,err2->column_name) - && strcompare(err1->cursor_name,err2->cursor_name) && strcompare(err1->message_text,err2->message)) - return true; - else - return false; -} - -void copyDiffErrorDataArea(ErrorDataArea *from, ErrorDataArea *to, ErrorData *edata) -{ - cleanErrorDataArea(to); - bool is_same = false; - MemoryContext oldcontext; - int count = 0; - - oldcontext = MemoryContextSwitchTo(u_sess->dolphin_errdata_ctx.dolphinErrorDataMemCxt); - - ListCell *lc = NULL; - lc = list_head(from->sqlErrorDataList); - foreach (lc, from->sqlErrorDataList) { - DolphinErrorData *eData = (DolphinErrorData *)lfirst(lc); - is_same = compareErrorData(eData, edata); - if (is_same) - continue; - DolphinErrorData *newErrData = (DolphinErrorData *)palloc(sizeof(DolphinErrorData)); - newErrData->elevel = eData->elevel; - newErrData->errorcode = pstrdup(eData->errorcode); - newErrData->sqlstatestr = pstrdup(eData->sqlstatestr); - newErrData->message_text = pstrdup(eData->message_text); - newErrData->class_origin = pstrdup(eData->class_origin); - newErrData->subclass_origin = pstrdup(eData->subclass_origin); - newErrData->constraint_catalog = pstrdup(eData->constraint_catalog); - newErrData->constraint_schema = pstrdup(eData->constraint_schema); - newErrData->constraint_name = pstrdup(eData->constraint_name); - newErrData->catalog_name = pstrdup(eData->catalog_name); - newErrData->schema_name = pstrdup(eData->schema_name); - newErrData->table_name = pstrdup(eData->table_name); - newErrData->column_name = pstrdup(eData->column_name); - newErrData->cursor_name = pstrdup(eData->cursor_name); - count++; - to->sqlErrorDataList = lappend(to->sqlErrorDataList, newErrData); - } - if (count != 0) { - to->current_edata_count = count; - for (int i = 0; i <= enum_dolphin_error_level::B_END; i++) { - to->current_edata_count_by_level[i] = from->current_edata_count_by_level[i]; - } - } - MemoryContextSwitchTo(oldcontext); -} void resetErrorDataArea(bool stacked, bool handler_active) { @@ -6014,9 +5935,10 @@ enum_dolphin_error_level errorLevelToDolphin(int elevel) void pushErrorData(ErrorData *edata) { MemoryContext oldcontext; - if (edata->is_signal == PLpgSQL_signal_resignal::PLPGSQL_RESIGNAL_WITHOUT_SQLSTATE) { + if (edata->is_signal == PLpgSQL_signal_resignal::PLPGSQL_RESIGNAL_WITHOUT_SQLSTATE || edata->is_signal == PLpgSQL_signal_resignal::PLPGSQL_NORMAL) { /* resignal without sqlstate */ - resetErrorDataArea(false, u_sess->dolphin_errdata_ctx.handler_active); + if (u_sess->dolphin_errdata_ctx.handler_active) + resetErrorDataArea(false, u_sess->dolphin_errdata_ctx.handler_active); } else if (edata->is_signal == PLpgSQL_signal_resignal::PLPGSQL_RESIGNAL_WITH_SQLSTATE) { /* resignal sqlstate */ copyErrorDataArea(u_sess->dolphin_errdata_ctx.lastErrorDataArea, u_sess->dolphin_errdata_ctx.errorDataArea); diff --git a/src/common/pl/plpgsql/src/pl_exec.cpp b/src/common/pl/plpgsql/src/pl_exec.cpp index c314f780f..87e5942cb 100644 --- a/src/common/pl/plpgsql/src/pl_exec.cpp +++ b/src/common/pl/plpgsql/src/pl_exec.cpp @@ -4126,9 +4126,7 @@ static int exec_stmt(PLpgSQL_execstate* estate, PLpgSQL_stmt* stmt, bool resigna } else { if ((enum PLpgSQL_stmt_types)stmt->cmd_type == PLPGSQL_STMT_BLOCK) { if (estate->handler_level == estate->block_level) { - copyDiffErrorDataArea(u_sess->dolphin_errdata_ctx.errorDataArea, u_sess->dolphin_errdata_ctx.lastErrorDataArea, estate->cur_error); - if (u_sess->dolphin_errdata_ctx.lastErrorDataArea->current_edata_count != 0) - copyErrorDataArea(u_sess->dolphin_errdata_ctx.lastErrorDataArea, u_sess->dolphin_errdata_ctx.errorDataArea); + copyErrorDataArea(u_sess->dolphin_errdata_ctx.errorDataArea, u_sess->dolphin_errdata_ctx.lastErrorDataArea); u_sess->dolphin_errdata_ctx.handler_active = false; } else if (!u_sess->dolphin_errdata_ctx.handler_active) { copyErrorDataArea(u_sess->dolphin_errdata_ctx.errorDataArea, u_sess->dolphin_errdata_ctx.lastErrorDataArea); diff --git a/src/include/utils/elog.h b/src/include/utils/elog.h index 78a474dc0..e918432ac 100644 --- a/src/include/utils/elog.h +++ b/src/include/utils/elog.h @@ -596,7 +596,6 @@ extern ErrorDataArea *initErrorDataArea(); extern void resetErrorDataArea(bool, bool); extern void pushErrorData(ErrorData *); extern void copyErrorDataArea(ErrorDataArea *from, ErrorDataArea *to); -extern void copyDiffErrorDataArea(ErrorDataArea *from, ErrorDataArea *to, ErrorData *edata); extern uint64 SqlErrorDataErrorCount(); extern uint64 SqlErrorDataWarnCount(); extern int SqlErrorDataCount(); diff --git a/src/test/regress/expected/mysql_condition.out b/src/test/regress/expected/mysql_condition.out index e8fa6682b..e764cf28b 100644 --- a/src/test/regress/expected/mysql_condition.out +++ b/src/test/regress/expected/mysql_condition.out @@ -703,7 +703,7 @@ NOTICE: current DA before mapped insert , error = 23502 , msg = null value in c NOTICE: stacked DA before mapped insert , error = 23502 , msg = null value in column "c1" violates not-null constraint NOTICE: current DA before mapped insert , num = 0 , errcount = 3 NOTICE: stacked DA before mapped insert , error = 23502 , msg = null value in column "c1" violates not-null constraint -NOTICE: current DA before mapped insert , num = 2 , errcount = 3 +NOTICE: current DA before mapped insert , num = 1 , errcount = 3 ERROR: GET STACKED DIAGNOSTICS cannot be used outside an exception handler CONTEXT: PL/pgSQL function prc() line 30 at GET DIAGNOSTICS show errors; @@ -747,7 +747,7 @@ NOTICE: current DA before mapped insert , error = 23502 , msg = null value in c NOTICE: stacked DA before mapped insert , error = 23502 , msg = null value in column "c1" violates not-null constraint NOTICE: current DA before mapped insert , num = 0 , errcount = 3 NOTICE: stacked DA before mapped insert , error = 23502 , msg = null value in column "c1" violates not-null constraint -NOTICE: current DA before mapped insert , num = 2 , errcount = -1 +NOTICE: current DA before mapped insert , num = 1 , errcount = -1 prc ----- diff --git a/src/test/regress/expected/mysql_resignal.out b/src/test/regress/expected/mysql_resignal.out index 255b8f767..4af2542a3 100644 --- a/src/test/regress/expected/mysql_resignal.out +++ b/src/test/regress/expected/mysql_resignal.out @@ -789,10 +789,11 @@ CONTEXT: PL/pgSQL function p1() line 4 at RESIGNAL (1 row) show warnings; - level | code | message ----------+------+---------------------- - Warning | 1120 | table is not defined -(1 row) + level | code | message +---------+-------+--------------------------- + Error | 42P01 | table "t1" does not exist + Warning | 1120 | table is not defined +(2 rows) DROP TABLE IF EXISTS t1; NOTICE: table "t1" does not exist, skipping From 16554827e33bf6aea4df8b816ec5122b8534538d Mon Sep 17 00:00:00 2001 From: nnuanyang Date: Fri, 15 Sep 2023 02:49:47 -0700 Subject: [PATCH 249/304] opt_interval ecpg bug --- src/common/backend/parser/gram.y | 1 + 1 file changed, 1 insertion(+) diff --git a/src/common/backend/parser/gram.y b/src/common/backend/parser/gram.y index 1ecf87960..3774cc94f 100644 --- a/src/common/backend/parser/gram.y +++ b/src/common/backend/parser/gram.y @@ -25594,6 +25594,7 @@ opt_evtime_unit: $$ = list_make1(makeIntConst(INTERVAL_MASK(YEAR) | INTERVAL_MASK(MONTH), @1)); } + ; opt_interval: YEAR_P From c4bd5e9098d98663d711e76cf27aeb8d0ee86650 Mon Sep 17 00:00:00 2001 From: wuyuechuan Date: Sat, 16 Sep 2023 15:04:47 +0800 Subject: [PATCH 250/304] validate dblink conn: prevent acess to unknown addresses --- contrib/dblink/dblink.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/contrib/dblink/dblink.cpp b/contrib/dblink/dblink.cpp index d1cc54ee6..842ab1bef 100644 --- a/contrib/dblink/dblink.cpp +++ b/contrib/dblink/dblink.cpp @@ -974,10 +974,10 @@ Datum dblink_open(PG_FUNCTION_ARGS) rconn = getConnectionByName(conname); } - linker = rconn->linker; - if (linker == NULL) { + if (rconn == NULL || rconn->linker == NULL) { DBLINK_CONN_NOT_AVAIL; } + linker = rconn->linker; /* Assemble sql */ appendStringInfo(&buf, "DECLARE %s CURSOR FOR %s", curname, sql); @@ -1028,10 +1028,10 @@ Datum dblink_close(PG_FUNCTION_ARGS) rconn = getConnectionByName(conname); } - linker = rconn->linker; - if (linker == NULL) { + if (rconn == NULL || rconn->linker == NULL) { DBLINK_CONN_NOT_AVAIL; } + linker = rconn->linker; appendStringInfo(&buf, "CLOSE %s", curname); @@ -3113,4 +3113,4 @@ static void storeRowInit(storeInfo* sinfo, int nfields, bool first) ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE); } -} \ No newline at end of file +} From 7c74605d38e8853c93fe0af0ef19b854f919ed65 Mon Sep 17 00:00:00 2001 From: yanghao Date: Fri, 15 Sep 2023 14:41:21 +0800 Subject: [PATCH 251/304] forbid use gsc in fenced udf mode --- src/common/backend/utils/cache/knl_localsysdbcache.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/common/backend/utils/cache/knl_localsysdbcache.cpp b/src/common/backend/utils/cache/knl_localsysdbcache.cpp index 160105aa1..448ca6bb1 100644 --- a/src/common/backend/utils/cache/knl_localsysdbcache.cpp +++ b/src/common/backend/utils/cache/knl_localsysdbcache.cpp @@ -147,6 +147,10 @@ void ResetDeepthInAcceptInvalidationMessage(int value) static bool SwitchToSessionSysCache() { + /* fenced mode not use gsc */ + if (FencedUDFMasterMode) { + return true; + } if ( #ifdef ENABLE_MULTIPLE_NODES /* ts code dont use gsc */ @@ -1214,4 +1218,4 @@ void TryFreshSmgrCache(struct SMgrRelationData *smgr) smgr->xact_seqno = t_thrd.lsc_cxt.xact_seqno; /* reset it and so caller will reload smgr_targblock */ smgr->smgr_targblock = InvalidBlockNumber; -} \ No newline at end of file +} From fedcde61f37e1c292f0578746a17d54cb335617d Mon Sep 17 00:00:00 2001 From: openGaussDev Date: Wed, 30 Aug 2023 11:48:40 +0800 Subject: [PATCH 252/304] fix problems of RTO STANDBY READ Offering: openGaussDevMore detail: fix problems of RTO STANDBY READ --- contrib/ndpplugin/common.h | 2 + contrib/pagehack/pagehack.cpp | 9 +- contrib/pg_xlogdump/pg_xlogdump.cpp | 7 +- src/bin/gs_guc/cluster_guc.conf | 8 +- src/common/backend/catalog/pg_partition.cpp | 3 +- src/common/backend/catalog/storage.cpp | 2 + src/common/backend/utils/cache/inval.cpp | 27 +- .../utils/cache/knl_globalsysdbcache.cpp | 4 +- src/common/backend/utils/cache/relcache.cpp | 6 +- src/common/backend/utils/cache/relmapper.cpp | 13 +- src/common/backend/utils/error/be_module.cpp | 2 +- .../backend/utils/misc/guc/guc_storage.cpp | 14 +- src/common/backend/utils/time/snapmgr.cpp | 52 +- .../optimizer/commands/dbcommands.cpp | 9 +- .../optimizer/commands/tablespace.cpp | 7 +- .../process/postmaster/postmaster.cpp | 48 +- src/gausskernel/process/stream/streamMain.cpp | 15 + src/gausskernel/process/tcop/postgres.cpp | 2 +- .../process/threadpool/knl_instance.cpp | 9 +- .../process/threadpool/knl_session.cpp | 3 + .../process/threadpool/knl_thread.cpp | 1 + .../process/threadpool/threadpool_worker.cpp | 4 + .../storage/access/heap/heapam.cpp | 7 +- .../storage/access/heap/tuptoaster.cpp | 6 +- .../storage/access/nbtree/nbtpage.cpp | 1 + .../storage/access/nbtree/nbtxlog.cpp | 2 +- .../storage/access/redo/redo_storage.cpp | 1 + .../storage/access/redo/redo_xlogutils.cpp | 27 +- .../storage/access/redo/standby_read/Makefile | 4 +- .../redo/standby_read/base_page_proc.cpp | 21 +- .../redo/standby_read/block_info_proc.cpp | 151 +++--- .../standby_read/lsn_info_double_list.cpp | 1 - .../redo/standby_read/lsn_info_proc.cpp | 71 ++- .../standby_read/standby_read_delay_ddl.cpp | 448 +++++++++++++++++ .../standby_read/standby_read_interface.cpp | 475 +++++++++++++++++- .../storage/access/spgist/spgxlog.cpp | 2 +- .../storage/access/transam/clog.cpp | 1 + .../access/transam/extreme_rto/dispatcher.cpp | 39 +- .../transam/extreme_rto/exrto_recycle.cpp | 15 +- .../access/transam/extreme_rto/page_redo.cpp | 309 ++++++++---- .../storage/access/transam/multi_redo_api.cpp | 3 +- .../access/transam/multi_redo_settings.cpp | 5 +- .../ondemand_extreme_rto/page_redo.cpp | 5 +- .../transam/parallel_recovery/dispatcher.cpp | 12 +- .../storage/access/transam/xact.cpp | 118 ++++- .../storage/access/transam/xlog.cpp | 84 ++-- .../storage/access/transam/xlogutils.cpp | 16 +- .../storage/access/ubtree/ubtxlog.cpp | 4 +- .../access/ustore/knl_uextremeredo.cpp | 38 +- .../storage/access/ustore/knl_utuptoaster.cpp | 4 +- .../storage/access/ustore/knl_uvisibility.cpp | 8 + .../access/ustore/undo/knl_uundoapi.cpp | 1 + .../access/ustore/undo/knl_uundorecycle.cpp | 25 +- .../access/ustore/undo/knl_uundozone.cpp | 39 +- src/gausskernel/storage/buffer/bufmgr.cpp | 132 +++-- src/gausskernel/storage/ipc/procarray.cpp | 60 ++- src/gausskernel/storage/ipc/sinval.cpp | 10 +- src/gausskernel/storage/ipc/sinvaladt.cpp | 24 +- src/gausskernel/storage/lmgr/lmgr.cpp | 8 +- src/gausskernel/storage/lmgr/lwlocknames.txt | 3 + src/gausskernel/storage/lmgr/proc.cpp | 7 +- src/gausskernel/storage/page/gs_xlogdump.cpp | 4 +- .../storage/replication/basebackup.cpp | 18 + .../heartbeat/libpq/fe-connect.cpp | 2 + src/gausskernel/storage/replication/slot.cpp | 1 + .../storage/replication/walreceiver.cpp | 1 + .../storage/replication/walsender.cpp | 1 + src/gausskernel/storage/smgr/md.cpp | 3 +- .../storage/smgr/storage_exrto_file.cpp | 181 ++++--- src/include/access/extreme_rto/dispatcher.h | 2 +- .../standby_read/block_info_meta.h | 6 +- .../standby_read/standby_read_base.h | 46 ++ .../standby_read/standby_read_delay_ddl.h | 40 ++ src/include/access/multi_redo_api.h | 9 +- src/include/access/rmgrlist.h | 2 +- .../access/ustore/undo/knl_uundozone.h | 4 +- src/include/access/xact.h | 6 +- src/include/access/xlogproc.h | 4 +- src/include/commands/dbcommands.h | 1 + .../knl/knl_guc/knl_instance_attr_storage.h | 2 +- src/include/knl/knl_instance.h | 15 +- src/include/knl/knl_session.h | 3 - src/include/knl/knl_thread.h | 15 +- src/include/storage/buf/bufmgr.h | 4 +- src/include/storage/lmgr.h | 3 +- src/include/storage/proc.h | 8 +- src/include/storage/procarray.h | 5 +- src/include/storage/sinval.h | 8 +- src/include/storage/sinvaladt.h | 2 +- src/include/storage/smgr/smgr.h | 2 +- src/include/utils/be_module.h | 2 + src/include/utils/inval.h | 3 +- src/include/utils/snapmgr.h | 3 + src/include/utils/snapshot.h | 7 +- src/test/ha/GNUmakefile | 3 + src/test/ha/ha_exrto_standby_read | 2 + src/test/ha/ha_schedule_single_standby_read | 1 + .../ha/results/exrtostandbyread/.gitignore | 4 + src/test/ha/run_ha_exrto_standby_read.sh | 59 +++ src/test/ha/run_ha_single_standby_read.sh | 50 ++ src/test/ha/standby_env.sh | 2 +- .../single_standby_read_base.sh | 144 ++++++ .../start_exrto_standby_read.sh | 81 +++ .../start_exrto_standby_read_multi_data.sh | 91 ++++ src/test/regress/CMakeLists.txt | 1 + src/test/regress/parallel_schedule0 | 2 +- src/test/regress/parallel_schedule0A | 2 +- src/test/regress/single_check.sh | 4 +- 108 files changed, 2652 insertions(+), 641 deletions(-) create mode 100644 src/gausskernel/storage/access/redo/standby_read/standby_read_delay_ddl.cpp create mode 100644 src/include/access/extreme_rto/standby_read/standby_read_delay_ddl.h create mode 100644 src/test/ha/ha_exrto_standby_read create mode 100644 src/test/ha/ha_schedule_single_standby_read create mode 100644 src/test/ha/results/exrtostandbyread/.gitignore create mode 100644 src/test/ha/run_ha_exrto_standby_read.sh create mode 100644 src/test/ha/run_ha_single_standby_read.sh create mode 100644 src/test/ha/testcase/exrtostandbyread/single_standby_read_base.sh create mode 100644 src/test/ha/testcase/exrtostandbyread/start_exrto_standby_read.sh create mode 100644 src/test/ha/testcase/exrtostandbyread/start_exrto_standby_read_multi_data.sh diff --git a/contrib/ndpplugin/common.h b/contrib/ndpplugin/common.h index 7291d0615..0631b34b2 100644 --- a/contrib/ndpplugin/common.h +++ b/contrib/ndpplugin/common.h @@ -242,6 +242,8 @@ typedef unsigned int Oid; #define InvalidOid 0 #define InvalidBktId (-1) +#define ExrtoReadStartLSNBktId (-5) +#define ExrtoReadEndLSNBktId (-6) typedef Oid regproc; typedef regproc RegProcedure; diff --git a/contrib/pagehack/pagehack.cpp b/contrib/pagehack/pagehack.cpp index 9c527326f..662b905c2 100644 --- a/contrib/pagehack/pagehack.cpp +++ b/contrib/pagehack/pagehack.cpp @@ -5172,7 +5172,7 @@ static bool parse_lsn_info_meta(const char *filename) } long pagenum = size / BLCKSZ; - fprintf(stdout, "file length is %ld, blknum is %ld\n", size, pagenum); + fprintf(stdout, "file length is %ld, pagenum is %ld\n", size, pagenum); for (loop = 1; loop <= pagenum; loop++) { fprintf(stdout, "Page %u information:\n", loop); @@ -5231,8 +5231,9 @@ static void parse_block_info_content(BlockMetaInfo *blockInfo) indents[indentLevel], blockInfo->timeline, blockInfo->record_num); fprintf(stdout, "%smin_lsn: %lu, max_lsn: %lu, flags: %u\n", indents[indentLevel], blockInfo->min_lsn, blockInfo->max_lsn, blockInfo->flags); - fprintf(stdout, "%slsn_info_list: prev %lu, next: %lu\n", - indents[indentLevel], blockInfo->lsn_info_list.prev, blockInfo->lsn_info_list.next); + fprintf(stdout, "%slsn_info_list: prev %lu, next: %lu. base_page_info_list: prev %lu, next: %lu\n", + indents[indentLevel], blockInfo->lsn_info_list.prev, blockInfo->lsn_info_list.next, + blockInfo->base_page_info_list.prev, blockInfo->base_page_info_list.next); } static bool parse_block_info_meta(const char *filename) @@ -5257,7 +5258,7 @@ static bool parse_block_info_meta(const char *filename) return false; } long pagenum = size / BLCKSZ; - fprintf(stdout, "file length is %ld, blknum is %ld\n", size, pagenum); + fprintf(stdout, "file length is %ld, pagenum is %ld\n", size, pagenum); for (loop = 0; loop < pagenum; loop++) { fprintf(stdout, "Page %u information:\n", loop); diff --git a/contrib/pg_xlogdump/pg_xlogdump.cpp b/contrib/pg_xlogdump/pg_xlogdump.cpp index 5d1ce64c9..9a33e9685 100644 --- a/contrib/pg_xlogdump/pg_xlogdump.cpp +++ b/contrib/pg_xlogdump/pg_xlogdump.cpp @@ -85,7 +85,7 @@ static void XLogDumpXLogRead(char* directory, TimeLineID timeline_id, XLogRecPtr static int XLogDumpReadPage(XLogReaderState* state, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetPtr, char* readBuff, TimeLineID* curFileTLI, char* xlog_path = NULL); static void XLogDumpCountRecord(XLogDumpConfig* config, XLogDumpStats* stats, XLogReaderState* record); -static void XLogDumpDisplayRecord(XLogDumpConfig* config, XLogReaderState* record); +void XLogDumpDisplayRecord(XLogDumpConfig* config, XLogReaderState* record); static void XLogDumpStatsRow(const char* name, uint64 n, uint64 total_count, uint64 rec_len, uint64 total_rec_len, uint64 fpi_len, uint64 total_fpi_len, uint64 tot_len, uint64 total_len); static void XLogDumpDisplayStats(XLogDumpConfig* config, XLogDumpStats* stats); @@ -534,7 +534,7 @@ static const char* XLogGetForkNames(ForkNumber forknum) /* * Print a record to stdout */ -static void XLogDumpDisplayRecord(XLogDumpConfig* config, XLogReaderState* record) +void XLogDumpDisplayRecord(XLogDumpConfig* config, XLogReaderState* record) { const RmgrDescData* desc = &RmgrDescTable[XLogRecGetRmid(record)]; RelFileNode rnode; @@ -577,7 +577,8 @@ static void XLogDumpDisplayRecord(XLogDumpConfig* config, XLogReaderState* recor XLogRecGetPhysicalBlock(record, block_id, &seg_fileno, &seg_blockno); // output format: ", blkref #%u: rel %u/%u/%u/%d storage %s fork %s blk %u (phy loc %u/%u) lastlsn %X/%X" - printf(", blkref #%d: rel %u/%u/%u", block_id, rnode.spcNode, rnode.dbNode, rnode.relNode); + printf(", blkref #%d: rel %u/%u/%u/%d/%d, forknum:%d", block_id, rnode.spcNode, rnode.dbNode, rnode.relNode, + rnode.bucketNode, rnode.opt, forknum); if (IsBucketFileNode(rnode)) { printf("/%d", rnode.bucketNode); } diff --git a/src/bin/gs_guc/cluster_guc.conf b/src/bin/gs_guc/cluster_guc.conf index b52f72f7f..68e265ef2 100755 --- a/src/bin/gs_guc/cluster_guc.conf +++ b/src/bin/gs_guc/cluster_guc.conf @@ -104,7 +104,7 @@ dirty_page_percent_max|real|0.1,1|NULL|NULL| group_concat_max_len|int64|0,9223372036854775807|NULL|NULL check_function_bodies|bool|0,0|NULL|NULL| checkpoint_completion_target|real|0,1|NULL|NULL| -standby_force_recyle_ratio|real|0,1|NULL|NULL| +standby_force_recycle_ratio|real|0,1|NULL|NULL| checkpoint_segments|int|1,2147483646|NULL|NULL| checkpoint_timeout|int|30,3600|s|NULL| checkpoint_warning|int|0,2147483647|s|NULL| @@ -272,7 +272,7 @@ hadr_max_size_for_xlog_receiver|int|0,2147483647|kB|NULL| hadr_recovery_time_target|int|0,3600|NULL|NULL| standby_recycle_interval|int|0,86400|s|NULL| standby_max_query_time|int|0,86400|s|NULL| -base_page_saved_interval|int|4,2000|NULL|NULL| +base_page_saved_interval|int|5,2000|NULL|NULL| hadr_recovery_point_target|int|0,3600|NULL|NULL| hadr_super_user_record_path|string|0,0|NULL|NULL| hll_default_log2m|int|10,16|NULL|NULL| @@ -713,8 +713,8 @@ undo_zone_count|int|0,1048576|NULL|NULL| stream_cluster_run_mode|enum|cluster_primary,cluster_standby|NULL|NULL| xlog_file_size|int64|1048576,576460752303423487|B|The value must be an integer multiple of 16777216(16M)| xlog_file_path|string|0,0|NULL|NULL| -max_standby_base_page_size|int64|0,576460752303423487|B|NULL| -max_standby_lsn_info_size|int64|0,576460752303423487|B|NULL| +max_standby_base_page_size|int64|1073741824,576460752303423487|B|NULL| +max_standby_lsn_info_size|int64|1073741824,576460752303423487|B|NULL| plsql_show_all_error|bool|0,0|NULL|NULL| partition_page_estimation|bool|0,0|NULL|NULL| enable_auto_clean_unique_sql|bool|0,0|NULL|NULL| diff --git a/src/common/backend/catalog/pg_partition.cpp b/src/common/backend/catalog/pg_partition.cpp index ed1dc256d..87a5401d2 100644 --- a/src/common/backend/catalog/pg_partition.cpp +++ b/src/common/backend/catalog/pg_partition.cpp @@ -25,6 +25,7 @@ */ #include "access/sysattr.h" +#include "access/multi_redo_api.h" #include "catalog/namespace.h" #include "catalog/pg_partition_fn.h" #include "catalog/pg_partition.h" @@ -1007,7 +1008,7 @@ List* searchPgPartitionByParentId(char parttype, Oid parentId, ScanDirection dir */ Snapshot snapshot = NULL; snapshot = SnapshotNow; - if (HistoricSnapshotActive()) { + if (HistoricSnapshotActive() || IS_EXRTO_RECOVERY_IN_PROGRESS) { snapshot = GetCatalogSnapshot(); } diff --git a/src/common/backend/catalog/storage.cpp b/src/common/backend/catalog/storage.cpp index 6d6d559fb..d27c6aa14 100644 --- a/src/common/backend/catalog/storage.cpp +++ b/src/common/backend/catalog/storage.cpp @@ -1269,7 +1269,9 @@ void xlog_block_smgr_redo_truncate(RelFileNode rnode, BlockNumber blkno, XLogRec smgrcreate(reln, MAIN_FORKNUM, true); UpdateMinRecoveryPoint(lsn, false); LockRelFileNode(rnode, AccessExclusiveLock); + (void)LWLockAcquire(RedoTruncateLock, LW_EXCLUSIVE); smgrtruncate(reln, MAIN_FORKNUM, blkno); + LWLockRelease(RedoTruncateLock); XLogTruncateRelation(rnode, MAIN_FORKNUM, blkno); Relation rel = CreateFakeRelcacheEntry(rnode); if (smgrexists(reln, FSM_FORKNUM)) diff --git a/src/common/backend/utils/cache/inval.cpp b/src/common/backend/utils/cache/inval.cpp index f50ca6122..e15dcdd28 100644 --- a/src/common/backend/utils/cache/inval.cpp +++ b/src/common/backend/utils/cache/inval.cpp @@ -117,6 +117,7 @@ #include "utils/syscache.h" #include "access/heapam.h" #include "catalog/pgxc_class.h" +#include "access/multi_redo_api.h" /* * To minimize palloc traffic, we keep pending requests in successively- @@ -909,6 +910,20 @@ void AcceptInvalidationMessages() --u_sess->inval_cxt.DeepthInAcceptInvalidationMessage; } +void reset_invalidation_cache() +{ + if (EnableLocalSysCache()) { + if (!IS_THREAD_POOL_WORKER) { + InvalidateSystemCaches(); + } else { + InvalidateThreadSystemCaches(); + InvalidateSessionSystemCaches(); + } + return; + } + InvalidateSystemCaches(); +} + /* * AtStart_Inval * Initialize inval lists at start of a main transaction. @@ -1057,12 +1072,16 @@ int xactGetCommittedInvalidationMessages(SharedInvalidationMessage** msgs, bool* * before and after we send the SI messages. See AtEOXact_Inval() */ void ProcessCommittedInvalidationMessages( - SharedInvalidationMessage* msgs, int nmsgs, bool RelcacheInitFileInval, Oid dbid, Oid tsid) + SharedInvalidationMessage* msgs, int nmsgs, bool RelcacheInitFileInval, Oid dbid, Oid tsid, XLogRecPtr lsn) { if (nmsgs <= 0) { return; } + if (!IS_EXRTO_READ) { + lsn = 0; + } + ereport(trace_recovery(DEBUG4), (errmsg("replaying commit with %d messages%s", nmsgs, @@ -1083,7 +1102,7 @@ void ProcessCommittedInvalidationMessages( u_sess->proc_cxt.DatabasePath = NULL; } - SendSharedInvalidMessages(msgs, nmsgs); + send_shared_invalid_messages(msgs, nmsgs, lsn); if (RelcacheInitFileInval) { RelationCacheInitFilePostInvalidate(); @@ -1565,13 +1584,13 @@ void CacheInvalidateSmgr(RelFileNodeBackend rnode) * should happen in low-level relmapper.c routines, which are executed while * replaying WAL as well as when creating it. */ -void CacheInvalidateRelmap(Oid databaseId) +void CacheInvalidateRelmap(Oid databaseId, XLogRecPtr lsn) { SharedInvalidationMessage msg; msg.rm.id = SHAREDINVALRELMAP_ID; msg.rm.dbId = databaseId; - SendSharedInvalidMessages(&msg, 1); + send_shared_invalid_messages(&msg, 1, lsn); } /* diff --git a/src/common/backend/utils/cache/knl_globalsysdbcache.cpp b/src/common/backend/utils/cache/knl_globalsysdbcache.cpp index 7f536f6c6..f740eebac 100644 --- a/src/common/backend/utils/cache/knl_globalsysdbcache.cpp +++ b/src/common/backend/utils/cache/knl_globalsysdbcache.cpp @@ -18,6 +18,7 @@ #include "utils/memutils.h" #include "utils/builtins.h" #include "access/xlog.h" +#include "access/multi_redo_api.h" #include "catalog/pg_authid.h" #include "catalog/pg_auth_members.h" #include "catalog/pg_database.h" @@ -682,7 +683,8 @@ void GlobalSysDBCache::InitSysCacheRelIds() void GlobalSysDBCache::RefreshHotStandby() { Assert(EnableGlobalSysCache()); - hot_standby = (t_thrd.postmaster_cxt.HaShmData->current_mode != STANDBY_MODE || XLogStandbyInfoActive()); + hot_standby = (t_thrd.postmaster_cxt.HaShmData->current_mode != STANDBY_MODE || (XLogStandbyInfoActive() && + !IsExtremeRedo())); if (hot_standby || !m_is_inited) { return; } diff --git a/src/common/backend/utils/cache/relcache.cpp b/src/common/backend/utils/cache/relcache.cpp index df44246d9..fa87ca9ef 100755 --- a/src/common/backend/utils/cache/relcache.cpp +++ b/src/common/backend/utils/cache/relcache.cpp @@ -40,6 +40,7 @@ #include "access/xact.h" #include "access/xlog.h" #include "access/multixact.h" +#include "access/multi_redo_api.h" #include "catalog/catalog.h" #include "catalog/heap.h" #include "catalog/catversion.h" @@ -1366,7 +1367,7 @@ HeapTuple ScanPgRelation(Oid targetRelId, bool indexOK, bool force_non_historic) * relfilenode of non mapped system relations during decoding. */ snapshot = SnapshotNow; - if (HistoricSnapshotActive() && !force_non_historic) { + if (HistoricSnapshotActive() && !force_non_historic || IS_EXRTO_RECOVERY_IN_PROGRESS) { snapshot = GetCatalogSnapshot(); } @@ -1601,7 +1602,7 @@ static void RelationBuildTupleDesc(Relation relation, bool onlyLoadInitDefVal) */ Snapshot snapshot = NULL; snapshot = SnapshotNow; - if (HistoricSnapshotActive()) { + if (HistoricSnapshotActive() || IS_EXRTO_RECOVERY_IN_PROGRESS) { snapshot = GetCatalogSnapshot(); } @@ -8741,4 +8742,3 @@ bool IsRelationReplidentKey(Relation r, int attno) RelationClose(idx_rel); return false; } - diff --git a/src/common/backend/utils/cache/relmapper.cpp b/src/common/backend/utils/cache/relmapper.cpp index 594ca8801..f8e482370 100644 --- a/src/common/backend/utils/cache/relmapper.cpp +++ b/src/common/backend/utils/cache/relmapper.cpp @@ -47,6 +47,7 @@ #include "access/xact.h" #include "access/xlog.h" #include "access/xloginsert.h" +#include "access/multi_redo_api.h" #include "catalog/catalog.h" #include "catalog/pg_tablespace.h" #include "catalog/storage.h" @@ -61,7 +62,7 @@ static void apply_map_update(RelMapFile* map, Oid relationId, Oid fileNode, bool add_okay); static void merge_map_updates(RelMapFile* map, const RelMapFile* updates, bool add_okay); static void write_relmap_file(bool shared, RelMapFile* newmap, bool write_wal, bool send_sinval, bool preserve_files, - Oid dbid, Oid tsid, const char* dbpath); + Oid dbid, Oid tsid, const char* dbpath, XLogRecPtr redo_lsn = 0); static void perform_relmap_update(bool shared, const RelMapFile* updates); static int WriteOldVersionRelmap(RelMapFile* map, int fd); static int ReadOldVersionRelmap(RelMapFile* map, int fd); @@ -742,7 +743,7 @@ void load_relmap_file(bool shared, RelMapFile *map) * map update could be happening. */ static void write_relmap_file(bool shared, RelMapFile* newmap, bool write_wal, bool send_sinval, bool preserve_files, - Oid dbid, Oid tsid, const char* dbpath) + Oid dbid, Oid tsid, const char* dbpath, XLogRecPtr redo_lsn) { int fd; RelMapFile* real_map = NULL; @@ -870,7 +871,7 @@ static void write_relmap_file(bool shared, RelMapFile* newmap, bool write_wal, b * as soon as others began to use the now-committed data. */ if (send_sinval) { - CacheInvalidateRelmap(dbid); + CacheInvalidateRelmap(dbid, redo_lsn); } /* * Make sure that the files listed in the map are not deleted if the outer @@ -1101,7 +1102,11 @@ void relmap_redo(XLogReaderState* record) */ XLogRecPtr lsn = record->EndRecPtr; UpdateMinRecoveryPoint(lsn, false); - write_relmap_file((xlrec->dbid == InvalidOid), &new_map, false, true, false, xlrec->dbid, xlrec->tsid, dbpath); + if (!IS_EXRTO_READ) { + lsn = 0; + } + write_relmap_file((xlrec->dbid == InvalidOid), &new_map, false, true, false, xlrec->dbid, xlrec->tsid, dbpath, + lsn); pfree_ext(dbpath); } else { ereport(PANIC, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("relmap_redo: unknown op code %u", info))); diff --git a/src/common/backend/utils/error/be_module.cpp b/src/common/backend/utils/error/be_module.cpp index 6d2d0d7e4..c564d5264 100755 --- a/src/common/backend/utils/error/be_module.cpp +++ b/src/common/backend/utils/error/be_module.cpp @@ -129,6 +129,7 @@ const module_data module_map[] = {{MOD_ALL, "ALL"}, {MOD_LOGICAL_DECODE, "LOGICAL_DECODE"}, {MOD_GPRC, "GPRC"}, {MOD_DISASTER_READ, "DISASTER_READ"}, + {MOD_STANDBY_READ, "STANDBY_READ"}, {MODE_REPSYNC, "REPSYNC"}, {MOD_SQLPATCH, "SQLPATCH"}, {MOD_DMS, "DMS"}, @@ -272,4 +273,3 @@ module_logging_enable_comm(ModuleId module_id) { enable_module_logging(module_id); } - diff --git a/src/common/backend/utils/misc/guc/guc_storage.cpp b/src/common/backend/utils/misc/guc/guc_storage.cpp index f8f7b9ee1..add3f8221 100755 --- a/src/common/backend/utils/misc/guc/guc_storage.cpp +++ b/src/common/backend/utils/misc/guc/guc_storage.cpp @@ -3263,6 +3263,7 @@ static void InitStorageConfigureNamesInt() NULL, NULL, NULL}, +#ifndef ENABLE_LITE_MODE {{"standby_recycle_interval", PGC_SIGHUP, NODE_ALL, @@ -3305,6 +3306,7 @@ static void InitStorageConfigureNamesInt() NULL, NULL, NULL}, +#endif {{"force_promote", PGC_POSTMASTER, NODE_ALL, @@ -3933,19 +3935,21 @@ static void InitStorageConfigureNamesReal() NULL, NULL, NULL}, - {{"standby_force_recyle_ratio", +#ifndef ENABLE_LITE_MODE + {{"standby_force_recycle_ratio", PGC_SIGHUP, NODE_ALL, RESOURCES_RECOVERY, gettext_noop("Sets the ratio that triggers forced recycling in extreme-rto standby read."), NULL}, - &g_instance.attr.attr_storage.standby_force_recyle_ratio, + &g_instance.attr.attr_storage.standby_force_recycle_ratio, 0.8, 0.0, 1.0, NULL, NULL, NULL}, +#endif {{"bypass_dram", PGC_SIGHUP, NODE_ALL, @@ -4096,6 +4100,7 @@ static void InitStorageConfigureNamesInt64() NULL, NULL, NULL}, +#ifndef ENABLE_LITE_MODE {{"max_standby_base_page_size", PGC_POSTMASTER, NODE_ALL, @@ -4104,7 +4109,7 @@ static void InitStorageConfigureNamesInt64() NULL}, &g_instance.attr.attr_storage.max_standby_base_page_size, INT64CONST(0x4000000000), /* 256GB */ - INT64CONST(0), + INT64CONST(0x40000000), /* 1GB */ INT64CONST(0x7FFFFFFFFFFFFFF), NULL, NULL, @@ -4117,11 +4122,12 @@ static void InitStorageConfigureNamesInt64() NULL}, &g_instance.attr.attr_storage.max_standby_lsn_info_size, INT64CONST(0x4000000000), /* 256GB */ - INT64CONST(0), + INT64CONST(0x40000000), /* 1GB */ INT64CONST(0x7FFFFFFFFFFFFFF), NULL, NULL, NULL}, +#endif /* End-of-list marker */ {{NULL, (GucContext)0, diff --git a/src/common/backend/utils/time/snapmgr.cpp b/src/common/backend/utils/time/snapmgr.cpp index 3a288e254..68edd053d 100644 --- a/src/common/backend/utils/time/snapmgr.cpp +++ b/src/common/backend/utils/time/snapmgr.cpp @@ -45,9 +45,11 @@ #include #include "access/csnlog.h" +#include "access/multi_redo_api.h" #include "access/transam.h" #include "access/twophase.h" #include "access/xact.h" +#include "access/multi_redo_api.h" #include "catalog/pg_type.h" #include "funcapi.h" #include "miscadmin.h" @@ -248,6 +250,10 @@ bool XidVisibleInSnapshot(TransactionId xid, Snapshot snapshot, TransactionIdSta ereport(FATAL, (errmsg("SS xid %lu's csn %lu is still COMMITTING after Master txn waited.", xid, csn))); } if (looped) { + /* don't change csn log in recovery */ + if (snapshot->takenDuringRecovery) { + return false; + } ereport(DEBUG1, (errmsg("transaction id %lu's csn %ld is changed to ABORT after lockwait.", xid, csn))); /* recheck if transaction id is finished */ RecheckXidFinish(xid, csn); @@ -281,9 +287,9 @@ bool XidVisibleInSnapshot(TransactionId xid, Snapshot snapshot, TransactionIdSta *sync = true; } if (TransactionIdIsValid(parentXid)) - SyncWaitXidEnd(parentXid, buffer); + SyncWaitXidEnd(parentXid, buffer, snapshot); else - SyncWaitXidEnd(xid, buffer); + SyncWaitXidEnd(xid, buffer, snapshot); looped = true; parentXid = InvalidTransactionId; goto loop; @@ -529,7 +535,7 @@ Snapshot GetTransactionSnapshot(bool force_local_snapshot) return u_sess->utils_cxt.CurrentSnapshot; } - if (IsolationUsesXactSnapshot()) { + if (IsolationUsesXactSnapshot()|| IS_EXRTO_STANDBY_READ) { #ifdef PGXC /* * Consider this test case taken from portals.sql @@ -554,12 +560,17 @@ Snapshot GetTransactionSnapshot(bool force_local_snapshot) if (IsConnFromCoord()) SnapshotSetCommandId(GetCurrentCommandId(false)); #endif + if (IS_EXRTO_STANDBY_READ) { + t_thrd.pgxact->xmin = u_sess->utils_cxt.CurrentSnapshot->xmin; + t_thrd.proc->exrto_min = u_sess->utils_cxt.CurrentSnapshot->read_lsn; + t_thrd.proc->exrto_read_lsn = t_thrd.proc->exrto_min; + t_thrd.proc->exrto_gen_snap_time = GetCurrentTimestamp(); + } return u_sess->utils_cxt.CurrentSnapshot; } Assert(!(u_sess->utils_cxt.CurrentSnapshot != NULL && u_sess->utils_cxt.CurrentSnapshot->user_data != NULL)); u_sess->utils_cxt.CurrentSnapshot = GetSnapshotData(u_sess->utils_cxt.CurrentSnapshotData, force_local_snapshot); - return u_sess->utils_cxt.CurrentSnapshot; } @@ -615,6 +626,16 @@ Snapshot GetLatestSnapshot(void) return u_sess->utils_cxt.SecondarySnapshot; } +Snapshot get_standby_snapshot() +{ + if (u_sess->utils_cxt.FirstSnapshotSet) { + Assert(!(u_sess->utils_cxt.CurrentSnapshot != NULL && u_sess->utils_cxt.CurrentSnapshot->user_data != NULL)); + return u_sess->utils_cxt.CurrentSnapshot; + } + + return GetTransactionSnapshot(); +} + /* * GetCatalogSnapshot * Get a snapshot that is sufficiently up-to-date for scan of the @@ -630,9 +651,27 @@ Snapshot GetCatalogSnapshot() if (HistoricSnapshotActive()) return u_sess->utils_cxt.HistoricSnapshot; + if (IS_EXRTO_RECOVERY_IN_PROGRESS && t_thrd.role != TRACK_STMT_WORKER && !dummyStandbyMode) { + return get_standby_snapshot(); + } + return SnapshotNow; } +/* + * get_toast_snapshot + * Get a snapshot that is sufficiently up-to-date for scan of the + * toast with the specified OID. + */ +Snapshot get_toast_snapshot() +{ + if (IS_EXRTO_RECOVERY_IN_PROGRESS && t_thrd.role != TRACK_STMT_WORKER && !dummyStandbyMode) { + return get_standby_snapshot(); + } + + return SnapshotToast; +} + /* * SnapshotSetCommandId * Propagate CommandCounterIncrement into the static snapshots, if set @@ -1225,6 +1264,11 @@ void AtEOXact_Snapshot(bool isCommit) u_sess->utils_cxt.FirstSnapshotSet = false; SnapshotResetXmin(); + t_thrd.proc->exrto_min = InvalidXLogRecPtr; + if (IS_EXRTO_STANDBY_READ && t_thrd.proc->exrto_reload_cache) { + t_thrd.proc->exrto_reload_cache = false; + reset_invalidation_cache(); + } } /* diff --git a/src/gausskernel/optimizer/commands/dbcommands.cpp b/src/gausskernel/optimizer/commands/dbcommands.cpp index 631a96836..57d17e67d 100644 --- a/src/gausskernel/optimizer/commands/dbcommands.cpp +++ b/src/gausskernel/optimizer/commands/dbcommands.cpp @@ -35,6 +35,7 @@ #include "access/multixact.h" #include "access/multi_redo_api.h" #include "access/extreme_rto/standby_read/block_info_meta.h" +#include "access/extreme_rto/standby_read/standby_read_delay_ddl.h" #include "catalog/catalog.h" #include "catalog/dependency.h" #include "catalog/indexing.h" @@ -2442,7 +2443,7 @@ void do_db_drop(Oid dbId, Oid tbSpcId) if (!rmtree(dst_path, true)) { ereport(WARNING, (errmsg("some useless files may be left behind in old database directory \"%s\"", dst_path))); } - if (IS_EXRTO_READ) { + if (RecoveryInProgress() && IS_EXRTO_READ) { /* remove file start with {db_id}_ */ extreme_rto_standby_read::remove_block_meta_info_files_of_db(dbId); } @@ -2489,7 +2490,11 @@ void xlogRemoveRemainSegsByDropDB(Oid dbId, Oid tablespaceId) void xlog_db_drop(XLogRecPtr lsn, Oid dbId, Oid tbSpcId) { UpdateMinRecoveryPoint(lsn, false); - do_db_drop(dbId, tbSpcId); + if (IS_EXRTO_READ) { + update_delay_ddl_db(dbId, tbSpcId, lsn); + } else { + do_db_drop(dbId, tbSpcId); + } xlogRemoveRemainSegsByDropDB(dbId, tbSpcId); } diff --git a/src/gausskernel/optimizer/commands/tablespace.cpp b/src/gausskernel/optimizer/commands/tablespace.cpp index 316bf02c8..dd36b8c32 100644 --- a/src/gausskernel/optimizer/commands/tablespace.cpp +++ b/src/gausskernel/optimizer/commands/tablespace.cpp @@ -55,6 +55,8 @@ #include "access/xact.h" #include "access/xlog.h" #include "access/xloginsert.h" +#include "access/multi_redo_api.h" +#include "access/extreme_rto/standby_read/standby_read_delay_ddl.h" #include "catalog/catalog.h" #include "catalog/dependency.h" #include "catalog/indexing.h" @@ -2618,9 +2620,12 @@ void xlog_drop_tblspc(Oid tsId) * etc etc. There's not much we can do about that, so just remove what * we can and press on. */ + if (!destroy_tablespace_directories(tsId, true)) { ResolveRecoveryConflictWithTablespace(tsId); - + if (IS_EXRTO_READ) { + delete_by_table_space(tsId); + } /* * If we did recovery processing then hopefully the backends who * wrote temp files should have cleaned up and exited by now. So diff --git a/src/gausskernel/process/postmaster/postmaster.cpp b/src/gausskernel/process/postmaster/postmaster.cpp index 7cb582977..3e518e813 100644 --- a/src/gausskernel/process/postmaster/postmaster.cpp +++ b/src/gausskernel/process/postmaster/postmaster.cpp @@ -234,6 +234,7 @@ #include "access/multi_redo_api.h" #include "postmaster/postmaster.h" #include "access/parallel_recovery/dispatcher.h" +#include "access/extreme_rto/standby_read/lsn_info_meta.h" #include "access/extreme_rto/standby_read/standby_read_base.h" #include "utils/distribute_test.h" #ifdef ENABLE_MULTIPLE_NODES @@ -315,8 +316,6 @@ extern void gs_set_hs_shm_data(HaShmemData* ha_shm_data); extern void ReaperBackendMain(); extern void AdjustThreadAffinity(); -extern void exrto_standby_read_init(); - #define EXTERN_SLOTS_NUM 17 volatile PMState pmState = PM_INIT; bool dummyStandbyMode = false; @@ -3347,12 +3346,29 @@ static void CheckExtremeRtoGUCConflicts(void) errhint("recommend config \"wal_receiver_buffer_size=64MB\""))); } +#ifdef ENABLE_LITE_MODE + if ((g_instance.attr.attr_storage.recovery_parse_workers > 1) && g_instance.attr.attr_storage.EnableHotStandby) { + ereport(ERROR, (errcode(ERRCODE_SYSTEM_ERROR), + errmsg("when enabling lite mode, extreme rto could not support hot standby."), + errhint("Either turn off extreme rto, or turn off hot_standby."))); + } +#endif + #ifndef ENABLE_MULTIPLE_NODES - if (IS_DISASTER_RECOVER_MODE &&(g_instance.attr.attr_storage.recovery_parse_workers > 1) && g_instance.attr.attr_storage.EnableHotStandby) { - ereport(ERROR, - (errcode(ERRCODE_SYSTEM_ERROR), - errmsg("For disaster standby cluster, extreme rto could not support hot standby."), - errhint("Either turn off extreme rto, or turn off hot_standby."))); + if (IS_DISASTER_RECOVER_MODE && (g_instance.attr.attr_storage.recovery_parse_workers > 1) && + g_instance.attr.attr_storage.EnableHotStandby) { + ereport(ERROR, (errcode(ERRCODE_SYSTEM_ERROR), + errmsg("For disaster standby cluster, extreme rto could not support hot standby."), + errhint("Either turn off extreme rto, or turn off hot_standby."))); + } + + if (g_instance.attr.attr_storage.EnableHotStandby == true) { + int base_page_saved_interval = g_instance.attr.attr_storage.base_page_saved_interval; + g_instance.attr.attr_storage.base_page_saved_interval = + (g_instance.attr.attr_storage.base_page_saved_interval / (int)extreme_rto_standby_read::LSN_NUM_PER_NODE) * + (int)extreme_rto_standby_read::LSN_NUM_PER_NODE; // Rounded down of 5 + ereport(LOG, (errmsg("base_page_saved_interval is %d, ori is %d.", + g_instance.attr.attr_storage.base_page_saved_interval, base_page_saved_interval))); } #endif @@ -4006,7 +4022,7 @@ static int ServerLoop(void) } } ADIO_END(); - + if (threadPoolActivated && (pmState == PM_RUN || pmState == PM_HOT_STANDBY)) g_threadPoolControler->AddWorkerIfNecessary(); @@ -5094,10 +5110,11 @@ int ProcessStartupPacket(Port* port, bool SSLdone) errmsg("can not accept connection in pending mode."))); } else { #ifdef ENABLE_MULTIPLE_NODES - if (STANDBY_MODE == hashmdata->current_mode && (!IS_MULTI_DISASTER_RECOVER_MODE || GTM_FREE_MODE || - (IS_PGXC_DATANODE && !g_instance.attr.attr_storage.EnableHotStandby))) { - ereport(ERROR, (errcode(ERRCODE_CANNOT_CONNECT_NOW), - errmsg("can not accept connection in standby mode."))); + if (STANDBY_MODE == hashmdata->current_mode && + (!IS_MULTI_DISASTER_RECOVER_MODE || GTM_FREE_MODE || + (IS_PGXC_DATANODE && !g_instance.attr.attr_storage.EnableHotStandby))) { + ereport(ERROR, + (errcode(ERRCODE_CANNOT_CONNECT_NOW), errmsg("can not accept connection in standby mode."))); } #else if (hashmdata->current_mode == STANDBY_MODE && !g_instance.attr.attr_storage.EnableHotStandby) { @@ -6963,9 +6980,10 @@ static void reaper(SIGNAL_ARGS) g_instance.pid_cxt.WalWriterAuxiliaryPID = initialize_util_thread(WALWRITERAUXILIARY); if (g_instance.pid_cxt.CBMWriterPID == 0 && !dummyStandbyMode && - u_sess->attr.attr_storage.enable_cbm_tracking) - - + u_sess->attr.attr_storage.enable_cbm_tracking) { + g_instance.pid_cxt.CBMWriterPID = initialize_util_thread(CBMWRITER); + } + if (IS_EXRTO_READ && g_instance.pid_cxt.exrto_recycler_pid == 0) { g_instance.pid_cxt.exrto_recycler_pid = initialize_util_thread(EXRTO_RECYCLER); } diff --git a/src/gausskernel/process/stream/streamMain.cpp b/src/gausskernel/process/stream/streamMain.cpp index 222e894b8..9cc8c2be5 100755 --- a/src/gausskernel/process/stream/streamMain.cpp +++ b/src/gausskernel/process/stream/streamMain.cpp @@ -24,7 +24,9 @@ #include "postgres.h" #include "access/gtm.h" +#include "access/multi_redo_api.h" #include "access/printtup.h" +#include "access/multi_redo_api.h" #include "distributelayer/streamMain.h" #include "distributelayer/streamProducer.h" #include "executor/exec/execStream.h" @@ -488,6 +490,19 @@ static void execute_stream_plan(StreamProducer* producer) * Start the portal. No parameters here. */ PortalStart(portal, producer->getParams(), 0, producer->getSnapShot()); + + /* The value of snapshot.read_lsn may be assigned to thread A and used on thread B. + So we should reassigned read_lsn to t_thrd of thread B */ + if (unlikely(IS_EXRTO_STANDBY_READ && producer->getSnapShot() != NULL)) { + t_thrd.proc->exrto_read_lsn = producer->getSnapShot()->read_lsn; + t_thrd.proc->exrto_min = t_thrd.proc->exrto_read_lsn; + } + + /* The value of snapshot.read_lsn may be assigned to thread A and used on thread B. + So we should reassigned read_lsn to t_thrd of thread B */ + if (unlikely(IS_EXRTO_STANDBY_READ && producer->getSnapShot() != NULL)) { + t_thrd.proc->exrto_read_lsn = producer->getSnapShot()->read_lsn; + } format = 0; PortalSetResultFormat(portal, 1, &format); diff --git a/src/gausskernel/process/tcop/postgres.cpp b/src/gausskernel/process/tcop/postgres.cpp index b08806c83..ced83182d 100755 --- a/src/gausskernel/process/tcop/postgres.cpp +++ b/src/gausskernel/process/tcop/postgres.cpp @@ -8583,7 +8583,7 @@ int PostgresMain(int argc, char* argv[], const char* dbname, const char* usernam lc_replan_nodegroup = InvalidOid; /* reset xmin before ReadCommand, in case blocking redo */ if (RecoveryInProgress()) { - t_thrd.pgxact->xmin = InvalidTransactionId; + } /* diff --git a/src/gausskernel/process/threadpool/knl_instance.cpp b/src/gausskernel/process/threadpool/knl_instance.cpp index 756c32229..a8ffe9555 100755 --- a/src/gausskernel/process/threadpool/knl_instance.cpp +++ b/src/gausskernel/process/threadpool/knl_instance.cpp @@ -42,6 +42,7 @@ #include "regex/regex.h" #include "utils/memutils.h" #include "utils/palloc.h" +#include "utils/snapshot.h" #include "workload/workload.h" #include "instruments/instr_waitevent.h" #include "access/multi_redo_api.h" @@ -325,7 +326,14 @@ static void knl_g_parallel_redo_init(knl_g_parallel_redo_context* predo_cxt) rc = memset_s(&predo_cxt->redoCpuBindcontrl, sizeof(RedoCpuBindControl), 0, sizeof(RedoCpuBindControl)); securec_check(rc, "", ""); predo_cxt->global_recycle_lsn = InvalidXLogRecPtr; + predo_cxt->exrto_snapshot = (ExrtoSnapshot)MemoryContextAllocZero( + INSTANCE_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_STORAGE), sizeof(ExrtoSnapshotData)); predo_cxt->redoItemHash = NULL; + + predo_cxt->standby_read_delay_ddl_stat.delete_stat = 0; + predo_cxt->standby_read_delay_ddl_stat.next_index_can_insert = 0; + predo_cxt->standby_read_delay_ddl_stat.next_index_need_unlink = 0; + predo_cxt->max_clog_pageno = 0; } static void knl_g_parallel_decode_init(knl_g_parallel_decode_context* pdecode_cxt) @@ -1071,4 +1079,3 @@ bool knl_g_get_redo_finish_status() uint32 isRedoFinish = pg_atomic_read_u32(&(g_instance.comm_cxt.predo_cxt.isRedoFinish)); return (isRedoFinish & REDO_FINISH_STATUS_CM) == REDO_FINISH_STATUS_CM; } - diff --git a/src/gausskernel/process/threadpool/knl_session.cpp b/src/gausskernel/process/threadpool/knl_session.cpp index 55b942021..63d72c182 100755 --- a/src/gausskernel/process/threadpool/knl_session.cpp +++ b/src/gausskernel/process/threadpool/knl_session.cpp @@ -957,6 +957,9 @@ static void knl_u_storage_init(knl_u_storage_context* storage_cxt) /* var in knl_uundofile.cpp */ storage_cxt->UndoFileCxt = NULL; + /* var in storage_exrto_file.cpp */ + storage_cxt->exrto_standby_read_file_cxt = NULL; + /* var in sync.cpp */ storage_cxt->pendingUnlinks = NIL; storage_cxt->pendingOpsCxt = NULL; diff --git a/src/gausskernel/process/threadpool/knl_thread.cpp b/src/gausskernel/process/threadpool/knl_thread.cpp index 4a604a239..0f46ecc62 100755 --- a/src/gausskernel/process/threadpool/knl_thread.cpp +++ b/src/gausskernel/process/threadpool/knl_thread.cpp @@ -962,6 +962,7 @@ static void knl_t_page_redo_init(knl_t_page_redo_context* page_redo_cxt) page_redo_cxt->sleep_long = false; page_redo_cxt->check_repair = false; page_redo_cxt->redo_worker_ptr = NULL; + page_redo_cxt->invalid_msg.valid = false; } static void knl_t_exrto_recycle_init(knl_t_exrto_recycle_context* exrto_recycle_cxt) diff --git a/src/gausskernel/process/threadpool/threadpool_worker.cpp b/src/gausskernel/process/threadpool/threadpool_worker.cpp index 0afdacf68..43c58ae3a 100644 --- a/src/gausskernel/process/threadpool/threadpool_worker.cpp +++ b/src/gausskernel/process/threadpool/threadpool_worker.cpp @@ -36,6 +36,7 @@ #include "threadpool/threadpool.h" #include "access/xact.h" +#include "access/multi_redo_api.h" #include "commands/prepare.h" #include "commands/tablespace.h" #include "commands/vacuum.h" @@ -534,6 +535,9 @@ void ThreadPoolWorker::CleanThread() thread_proc->workingVersionNum = pg_atomic_read_u32(&WorkingGrandVersionNum); if (m_currentSession != NULL) { + if (IS_EXRTO_STANDBY_READ) { + AtEOXact_Snapshot(false); + } DetachSessionFromThread(); } } diff --git a/src/gausskernel/storage/access/heap/heapam.cpp b/src/gausskernel/storage/access/heap/heapam.cpp index cda1455f3..5625c0d60 100755 --- a/src/gausskernel/storage/access/heap/heapam.cpp +++ b/src/gausskernel/storage/access/heap/heapam.cpp @@ -8737,7 +8737,11 @@ static void heap_xlog_cleanup_info(XLogReaderState* record) RelFileNode tmp_node; RelFileNodeCopy(tmp_node, xlrec->node, XLogRecGetBucketId(record)); - if (InHotStandby && g_supportHotStandby) { + if (IsExtremeRedo()) { + return; + } + + if (InHotStandby && g_supportHotStandby && !IS_EXRTO_READ) { XLogRecPtr lsn = record->EndRecPtr; ResolveRecoveryConflictWithSnapshot(xlrec->latestRemovedXid, tmp_node, lsn); } @@ -10324,4 +10328,3 @@ HeapTuple heapam_index_fetch_tuple(IndexScanDesc scan, bool *all_dead, bool* has return NULL; } - diff --git a/src/gausskernel/storage/access/heap/tuptoaster.cpp b/src/gausskernel/storage/access/heap/tuptoaster.cpp index 580b08f28..b405a8954 100644 --- a/src/gausskernel/storage/access/heap/tuptoaster.cpp +++ b/src/gausskernel/storage/access/heap/tuptoaster.cpp @@ -2380,7 +2380,7 @@ struct varlena* heap_internal_toast_fetch_datum(struct varatt_external toast_poi */ nextidx = 0; - toastscan = systable_beginscan_ordered(toastrel, toastidx, SnapshotToast, 1, &toastkey); + toastscan = systable_beginscan_ordered(toastrel, toastidx, get_toast_snapshot(), 1, &toastkey); while ((ttup = systable_getnext_ordered(toastscan, ForwardScanDirection)) != NULL) { /* * Have a chunk, extract the sequence number and the data @@ -2559,7 +2559,7 @@ struct varlena* HeapInternalToastFetchDatumSlice(struct varatt_external toastPoi * The index is on (valueid, chunkidx) so they will come in order */ nextidx = startchunk; - toastscan = systable_beginscan_ordered(toastrel, toastidx, SnapshotToast, nscankeys, toastkey); + toastscan = systable_beginscan_ordered(toastrel, toastidx, get_toast_snapshot(), nscankeys, toastkey); while ((ttup = systable_getnext_ordered(toastscan, ForwardScanDirection)) != NULL) { /* * Have a chunk, extract the sequence number and the data @@ -2775,7 +2775,7 @@ static struct varlena *toast_huge_fetch_datum_slice(struct varlena *attr, int64 Relation toastidx = index_open(toastrel->rd_rel->reltoastidxid, AccessShareLock); ScanKeyInit(&toastkey, (AttrNumber)1, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(large_toast_pointer.va_valueid)); - toastscan = systable_beginscan_ordered(toastrel, toastidx, SnapshotToast, 1, &toastkey); + toastscan = systable_beginscan_ordered(toastrel, toastidx, get_toast_snapshot(), 1, &toastkey); while ((ttup = systable_getnext_ordered(toastscan, ForwardScanDirection)) != NULL) { chunk = DatumGetPointer(fastgetattr(ttup, CHUNK_DATA_ATTR, toast_tup_desc, &isnull)); struct varatt_external toast_pointer; diff --git a/src/gausskernel/storage/access/nbtree/nbtpage.cpp b/src/gausskernel/storage/access/nbtree/nbtpage.cpp index 5d6c20ee5..8173ccb92 100644 --- a/src/gausskernel/storage/access/nbtree/nbtpage.cpp +++ b/src/gausskernel/storage/access/nbtree/nbtpage.cpp @@ -25,6 +25,7 @@ #include "knl/knl_variable.h" #include "access/hio.h" +#include "access/multi_redo_api.h" #include "access/nbtree.h" #include "access/transam.h" #include "access/visibilitymap.h" diff --git a/src/gausskernel/storage/access/nbtree/nbtxlog.cpp b/src/gausskernel/storage/access/nbtree/nbtxlog.cpp index 066b2e575..b0277c08c 100755 --- a/src/gausskernel/storage/access/nbtree/nbtxlog.cpp +++ b/src/gausskernel/storage/access/nbtree/nbtxlog.cpp @@ -1072,7 +1072,7 @@ static void btree_xlog_reuse_page(XLogReaderState *record) RelFileNode tmp_node; RelFileNodeCopy(tmp_node, xlrec->node, XLogRecGetBucketId(record)); - if (InHotStandby && g_supportHotStandby) { + if (InHotStandby && g_supportHotStandby && !IS_EXRTO_READ) { XLogRecPtr lsn = record->EndRecPtr; ResolveRecoveryConflictWithSnapshot(xlrec->latestRemovedXid, tmp_node, lsn); } diff --git a/src/gausskernel/storage/access/redo/redo_storage.cpp b/src/gausskernel/storage/access/redo/redo_storage.cpp index 73a25fbec..7288efa5b 100644 --- a/src/gausskernel/storage/access/redo/redo_storage.cpp +++ b/src/gausskernel/storage/access/redo/redo_storage.cpp @@ -80,6 +80,7 @@ XLogRecParseState *smgr_redo_parse_to_block(XLogReaderState *record, uint32 *blo *blocknum = 0; if ((info == XLOG_SMGR_CREATE) || (info == XLOG_SMGR_TRUNCATE)) { recordstatehead = smgr_xlog_relnode_parse_to_block(record, blocknum); + recordstatehead->isFullSync = record->isFullSync; } else { ereport(PANIC, (errmsg("smgr_redo_parse_to_block: unknown op code %u", info))); } diff --git a/src/gausskernel/storage/access/redo/redo_xlogutils.cpp b/src/gausskernel/storage/access/redo/redo_xlogutils.cpp index cf6feafb7..53f86af4d 100644 --- a/src/gausskernel/storage/access/redo/redo_xlogutils.cpp +++ b/src/gausskernel/storage/access/redo/redo_xlogutils.cpp @@ -141,13 +141,13 @@ bool DoLsnCheck(const RedoBufferInfo *bufferinfo, bool willInit, XLogRecPtr last *needRepair = true; XLogLsnCheckLogInvalidPage(bufferinfo, LSN_CHECK_ERROR, pblk); } - ereport(elevel, - (errmsg("lsn check error, record last lsn (%X/%X) ,lsn in current page %X/%X, " - "page info:%u/%u/%u forknum %d blknum:%u lsn %X/%X", - (uint32)(lastLsn >> XLOG_LSN_SWAP), (uint32)(lastLsn), (uint32)(pageCurLsn >> XLOG_LSN_SWAP), - (uint32)(pageCurLsn), blockinfo->rnode.spcNode, blockinfo->rnode.dbNode, - blockinfo->rnode.relNode, blockinfo->forknum, blockinfo->blkno, (uint32)(lsn >> XLOG_LSN_SWAP), - (uint32)(lsn)))); + ereport(elevel, (errmsg("lsn check error, record last lsn (%X/%X) ,lsn in current page %X/%X, " + "page info:%u/%u/%u/%d/%d forknum %d blknum:%u lsn %X/%X", + (uint32)(lastLsn >> XLOG_LSN_SWAP), (uint32)(lastLsn), + (uint32)(pageCurLsn >> XLOG_LSN_SWAP), (uint32)(pageCurLsn), + blockinfo->rnode.spcNode, blockinfo->rnode.dbNode, blockinfo->rnode.relNode, + blockinfo->rnode.bucketNode, blockinfo->rnode.opt, blockinfo->forknum, + blockinfo->blkno, (uint32)(lsn >> XLOG_LSN_SWAP), (uint32)(lsn)))); return false; } } @@ -1769,6 +1769,9 @@ bool XLogBlockRedoForExtremeRTO(XLogRecParseState *redoblocktate, RedoBufferInfo if ((block_valid != BLOCK_DATA_UNDO_TYPE) && g_instance.attr.attr_storage.EnableHotStandby && IsDefaultExtremeRtoMode() && XLByteLT(PageGetLSN(bufferinfo->pageinfo.page), blockhead->end_ptr)) { + if (bufferinfo->blockinfo.forknum >= EXRTO_FORK_NUM) { + ereport(PANIC, (errmsg("forknum is illegal: %d", bufferinfo->blockinfo.forknum))); + } BufferTag buf_tag; INIT_BUFFERTAG(buf_tag, bufferinfo->blockinfo.rnode, bufferinfo->blockinfo.forknum, bufferinfo->blockinfo.blkno); @@ -1951,7 +1954,7 @@ void redo_target_page(const BufferTag &buf_tag, StandbyReadLsnInfoArray *lsn_inf /* do we need register interrupt func here? like ProcessConfigFile */ XLogParseBufferInitFunc(&redo_pm, MAX_BUFFER_NUM_PER_WAL_RECORD, NULL, NULL); if (xlog_reader == NULL) { - ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"), + ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("redo_target_page: out of memory"), errdetail("Failed while allocating an XLog reading processor."))); } @@ -1961,7 +1964,7 @@ void redo_target_page(const BufferTag &buf_tag, StandbyReadLsnInfoArray *lsn_inf XLogRecord *record = XLogReadRecord(xlog_reader, lsn_info->lsn_array[i], &error_msg); if (record == NULL) { ereport(ERROR, (errcode_for_file_access(), - errmsg("could not read two-phase state from xlog at %X/%X, errormsg: %s", + errmsg("redo_target_page: could not read wal record from xlog at %X/%X, errormsg: %s", (uint32)(lsn_info->lsn_array[i] >> LSN_MOVE32), (uint32)(lsn_info->lsn_array[i]), error_msg ? error_msg : " "))); } @@ -1970,12 +1973,12 @@ void redo_target_page(const BufferTag &buf_tag, StandbyReadLsnInfoArray *lsn_inf XLogRecParseState *state = XLogParseToBlockCommonFunc(xlog_reader, &num); if (num == 0) { - ereport(ERROR, (errmsg("internal error, xlog in lsn %X/%X doesn't contain any block.", + ereport(ERROR, (errmsg("redo_target_page: internal error, xlog in lsn %X/%X doesn't contain any block.", (uint32)(lsn_info->lsn_array[i] >> LSN_MOVE32), (uint32)(lsn_info->lsn_array[i])))); } if (state == NULL) { - ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"), + ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("redo_target_page: out of memory"), errdetail("Failed while wal parse to block."))); } XLogRecParseState *state_iter = state; @@ -1986,7 +1989,7 @@ void redo_target_page(const BufferTag &buf_tag, StandbyReadLsnInfoArray *lsn_inf state_iter = (XLogRecParseState *)(state_iter->nextrecord); } if (state_iter == NULL) { - ereport(ERROR, (errmsg("internal error, xlog in lsn %X/%X doesn't contain target block.", + ereport(ERROR, (errmsg("redo_target_page: internal error, xlog in lsn %X/%X doesn't contain target block.", (uint32)(lsn_info->lsn_array[i] >> LSN_MOVE32), (uint32)(lsn_info->lsn_array[i])))); } buf_info.lsn = state_iter->blockparse.blockhead.end_ptr; diff --git a/src/gausskernel/storage/access/redo/standby_read/Makefile b/src/gausskernel/storage/access/redo/standby_read/Makefile index 236784529..9d1fc6488 100644 --- a/src/gausskernel/storage/access/redo/standby_read/Makefile +++ b/src/gausskernel/storage/access/redo/standby_read/Makefile @@ -17,7 +17,7 @@ # Makefile for access/psort # # IDENTIFICATION -# src/backend/access/psort/Makefile +# src/gausskernel/storage/access/redo/standby_read/Makefile # #------------------------------------------------------------------------- @@ -32,6 +32,6 @@ ifneq "$(MAKECMDGOALS)" "clean" endif endif endif -OBJS = base_page_proc.o block_info_proc.o lsn_info_double_list.o lsn_info_proc.o standby_read_interface.o +OBJS = base_page_proc.o block_info_proc.o lsn_info_double_list.o lsn_info_proc.o standby_read_interface.o standby_read_delay_ddl.o include $(top_srcdir)/src/gausskernel/common.mk diff --git a/src/gausskernel/storage/access/redo/standby_read/base_page_proc.cpp b/src/gausskernel/storage/access/redo/standby_read/base_page_proc.cpp index 514457106..b7f6ee289 100644 --- a/src/gausskernel/storage/access/redo/standby_read/base_page_proc.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/base_page_proc.cpp @@ -65,7 +65,11 @@ void generate_base_page(StandbyReadMetaInfo* meta_info, const Page src_page) Buffer dest_buf = buffer_read_base_page(meta_info->batch_id, meta_info->redo_id, position, RBM_ZERO_AND_LOCK); +#ifdef ENABLE_UT + Page dest_page = get_page_from_buffer(dest_buf); +#else Page dest_page = BufferGetPage(dest_buf); +#endif errno_t rc = memcpy_s(dest_page, BLCKSZ, src_page, BLCKSZ); securec_check(rc, "\0", "\0"); MarkBufferDirty(dest_buf); @@ -77,17 +81,21 @@ void generate_base_page(StandbyReadMetaInfo* meta_info, const Page src_page) void read_base_page(const BufferTag& buf_tag, BasePagePosition position, BufferDesc* dest_buf_desc) { extreme_rto::RedoItemTag redo_item_tag; - const uint32 worker_num_per_mng = extreme_rto::get_page_redo_worker_num_per_manager(); + INIT_REDO_ITEM_TAG(redo_item_tag, buf_tag.rnode, buf_tag.forkNum, buf_tag.blockNum); + const uint32 worker_num_per_mng = extreme_rto::get_page_redo_worker_num_per_manager(); /* batch id and worker id start from 1 when reading a page */ - uint32 batch_id = extreme_rto::GetSlotId(buf_tag.rnode, 0, 0, extreme_rto::GetBatchCount()) + 1; - INIT_REDO_ITEM_TAG(redo_item_tag, buf_tag.rnode, buf_tag.forkNum, buf_tag.blockNum); + uint32 batch_id = extreme_rto::GetSlotId(buf_tag.rnode, 0, 0, (uint32)extreme_rto::get_batch_redo_num()) + 1; uint32 redo_worker_id = extreme_rto::GetWorkerId(&redo_item_tag, worker_num_per_mng) + 1; Buffer buffer = buffer_read_base_page(batch_id, redo_worker_id, position, RBM_NORMAL); LockBuffer(buffer, BUFFER_LOCK_SHARE); +#ifdef ENABLE_UT + Page src_page = get_page_from_buffer(buffer); +#else Page src_page = BufferGetPage(buffer); +#endif Size page_size = BufferGetPageSize(buffer); Page dest_page = (Page)BufHdrGetBlock(dest_buf_desc); errno_t rc = memcpy_s(dest_page, page_size, src_page, page_size); @@ -103,5 +111,12 @@ void recycle_base_page_file(uint32 batch_id, uint32 redo_id, BasePagePosition re smgrdounlink(smgr, true, (BlockNumber)(recycle_pos / BLCKSZ)); } +#ifdef ENABLE_UT +Page get_page_from_buffer(Buffer buf) +{ + return BufferGetPage(buf); +} +#endif + } // namespace extreme_rto_standby_read diff --git a/src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp b/src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp index 356a6687b..08fefe6ce 100644 --- a/src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp @@ -16,7 +16,7 @@ * block_info_proc.cpp * * IDENTIFICATION - * src/gausskernel/storage/recovery/parallel/blocklevel/standby_read/block_info_proc.cpp + * src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp * * ------------------------------------------------------------------------- */ @@ -24,6 +24,7 @@ #include #include "access/extreme_rto/standby_read/block_info_meta.h" #include "access/extreme_rto/standby_read/lsn_info_meta.h" +#include "access/extreme_rto/standby_read/standby_read_base.h" #include "storage/smgr/relfilenode.h" namespace extreme_rto_standby_read { @@ -33,7 +34,7 @@ void block_info_page_init(Page page) static_assert(sizeof(BlockInfoPageHeader) == BLOCK_INFO_HEAD_SIZE, "BlockInfoPageHeader size is not 64 bytes"); static_assert(sizeof(BlockMetaInfo) == BLOCK_INFO_SIZE, "BlockMetaInfo size is not 64 bytes"); - BlockInfoPageHeader* page_header = (BlockInfoPageHeader*)page; + BlockInfoPageHeader *page_header = (BlockInfoPageHeader *)page; errno_t ret = memset_s(page_header, BLCKSZ, 0, BLCKSZ); securec_check(ret, "", ""); page_header->flags |= BLOCK_INFO_PAGE_VALID_FLAG; @@ -44,15 +45,21 @@ inline BlockNumber data_block_number_to_meta_page_number(BlockNumber block_num) { return block_num / BLOCK_INFO_NUM_PER_PAGE; } - +#ifdef ENABLE_UT +uint32 block_info_meta_page_offset(BlockNumber block_num) +{ + return (block_num % BLOCK_INFO_NUM_PER_PAGE) * BLOCK_INFO_SIZE + BLOCK_INFO_HEAD_SIZE; +} +#else inline uint32 block_info_meta_page_offset(BlockNumber block_num) { return (block_num % BLOCK_INFO_NUM_PER_PAGE) * BLOCK_INFO_SIZE + BLOCK_INFO_HEAD_SIZE; } +#endif // get page, just have pin, no lock BlockMetaInfo* get_block_meta_info_by_relfilenode( - const BufferTag& buf_tag, BufferAccessStrategy strategy, ReadBufferMode mode, Buffer* buffer) + const BufferTag& buf_tag, BufferAccessStrategy strategy, ReadBufferMode mode, Buffer* buffer, bool need_share_lock) { RelFileNode standby_read_rnode = buf_tag.rnode; standby_read_rnode.spcNode = EXRTO_BLOCK_INFO_SPACE_OID; @@ -63,25 +70,47 @@ BlockMetaInfo* get_block_meta_info_by_relfilenode( *buffer = ReadBuffer_common(smgr, 0, buf_tag.forkNum, meta_block_num, mode, strategy, &hit, NULL); if (*buffer == InvalidBuffer) { + ereport(DEBUG1, (errmodule(MOD_STANDBY_READ), + errmsg("get block meta info failed, buffer invalid %u/%u/%u %d %u, meta_block_num %u", + buf_tag.rnode.spcNode, buf_tag.rnode.dbNode, buf_tag.rnode.relNode, buf_tag.forkNum, + buf_tag.blockNum, meta_block_num))); return NULL; } + if (need_share_lock) { + LockBuffer(*buffer, BUFFER_LOCK_SHARE); + } else { + LockBuffer(*buffer, BUFFER_LOCK_EXCLUSIVE); + } + +#ifdef ENABLE_UT + Page page = get_page_from_buffer(*buffer); +#else Page page = BufferGetPage(*buffer); - if (!is_block_info_page_valid((BlockInfoPageHeader*)page)) { +#endif + if (!is_block_info_page_valid((BlockInfoPageHeader *)page)) { if (mode == RBM_NORMAL) { - ReleaseBuffer(*buffer); + UnlockReleaseBuffer(*buffer); + ereport(DEBUG1, (errmodule(MOD_STANDBY_READ), + errmsg("get block meta info failed, page invalid %u/%u/%u %d %u, meta_block_num %u", + buf_tag.rnode.spcNode, buf_tag.rnode.dbNode, buf_tag.rnode.relNode, buf_tag.forkNum, + buf_tag.blockNum, meta_block_num))); return NULL; } } uint32 offset = block_info_meta_page_offset(buf_tag.blockNum); - BlockMetaInfo *block_info = ((BlockMetaInfo*)(page + offset)); + BlockMetaInfo *block_info = ((BlockMetaInfo *)(page + offset)); if (!is_block_meta_info_valid(block_info) && mode == RBM_NORMAL) { - ReleaseBuffer(*buffer); - + ereport(DEBUG1, + (errmsg("block_info is invalid %u/%u/%u %d %u min lsn %08X/%08X max lsn %08X/%08X flags:%u", + buf_tag.rnode.spcNode, buf_tag.rnode.dbNode, buf_tag.rnode.relNode, buf_tag.forkNum, + buf_tag.blockNum, (uint32)(block_info->min_lsn >> UINT64_HALF), (uint32)block_info->min_lsn, + (uint32)(block_info->max_lsn >> UINT64_HALF), (uint32)block_info->max_lsn, block_info->flags))); + UnlockReleaseBuffer(*buffer); return NULL; } - + Assert(block_info != NULL); return block_info; } @@ -106,11 +135,14 @@ void insert_lsn_to_block_info( ereport(PANIC, (errmsg("insert lsn failed,block invalid %u/%u/%u %d %u", buf_tag.rnode.spcNode, buf_tag.rnode.dbNode, buf_tag.rnode.relNode, buf_tag.forkNum, buf_tag.blockNum))); } - LockBuffer(block_info_buf, BUFFER_LOCK_EXCLUSIVE); +#ifdef ENABLE_UT + Page page = get_page_from_buffer(block_info_buf); +#else Page page = BufferGetPage(block_info_buf); +#endif XLogRecPtr current_page_lsn = PageGetLSN(base_page); if (!is_block_meta_info_valid(block_info)) { - if (!is_block_info_page_valid((BlockInfoPageHeader*)page)) { + if (!is_block_info_page_valid((BlockInfoPageHeader *)page)) { block_info_page_init(page); } @@ -142,14 +174,19 @@ StandbyReadRecyleState recyle_block_info( BlockMetaInfo* block_meta_info = get_block_meta_info_by_relfilenode(buf_tag, NULL, RBM_NORMAL, &buffer); if ((block_meta_info == NULL) || (buffer == InvalidBuffer)) { // no block info, should not at this branch - ereport(WARNING, (errmsg("block meta is invalid %u/%u/%u %d %u", buf_tag.rnode.spcNode, buf_tag.rnode.dbNode, - buf_tag.rnode.relNode, buf_tag.forkNum, buf_tag.blockNum))); + ereport(WARNING, (errmodule(MOD_STANDBY_READ), errmsg("block meta is invalid %u/%u/%u %d %u", + buf_tag.rnode.spcNode, buf_tag.rnode.dbNode, + buf_tag.rnode.relNode, buf_tag.forkNum, buf_tag.blockNum))); return STANDBY_READ_RECLYE_ALL; } StandbyReadRecyleState stat = STANDBY_READ_RECLYE_NONE; - LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); Assert(((block_meta_info->flags & BLOCK_INFO_NODE_VALID_FLAG) == BLOCK_INFO_NODE_VALID_FLAG)); if (XLByteLT(block_meta_info->max_lsn, recyle_lsn)) { + ereport(DEBUG1, + (errmsg(EXRTOFORMAT("block meta recycle all %u/%u/%u %d %u, max lsn %08X/%08X, recycle lsn %08X/%08X"), + buf_tag.rnode.spcNode, buf_tag.rnode.dbNode, buf_tag.rnode.relNode, buf_tag.forkNum, + buf_tag.blockNum, (uint32)(block_meta_info->max_lsn >> UINT64_HALF), + (uint32)block_meta_info->max_lsn, (uint32)(recyle_lsn >> UINT64_HALF), (uint32)recyle_lsn))); block_meta_info->flags &= ~BLOCK_INFO_NODE_VALID_FLAG; stat = STANDBY_READ_RECLYE_ALL; MarkBufferDirty(buffer); @@ -186,13 +223,11 @@ bool get_page_lsn_info(const BufferTag& buf_tag, BufferAccessStrategy strategy, StandbyReadLsnInfoArray* lsn_info) { Buffer buf; - BlockMetaInfo* block_meta_info = get_block_meta_info_by_relfilenode(buf_tag, strategy, RBM_NORMAL, &buf); + BlockMetaInfo* block_meta_info = get_block_meta_info_by_relfilenode(buf_tag, strategy, RBM_NORMAL, &buf, true); if (block_meta_info == NULL) { return false; } - LockBuffer(buf, BUFFER_LOCK_SHARE); - if (XLByteLT(read_lsn, block_meta_info->min_lsn)) { UnlockReleaseBuffer(buf); ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), @@ -203,9 +238,22 @@ bool get_page_lsn_info(const BufferTag& buf_tag, BufferAccessStrategy strategy, } Assert(block_meta_info->base_page_info_list.prev != LSN_INFO_LIST_HEAD); + if (block_meta_info->base_page_info_list.prev == LSN_INFO_LIST_HEAD) { + ereport(ERROR, + ((errmsg("block_meta_info->base_page_info_list.prev is invaild. timeline %u, recordnum %u , min lsn " + "%lu, max lsn %lu, read lsn %lu", + block_meta_info->timeline, block_meta_info->record_num, block_meta_info->min_lsn, + block_meta_info->max_lsn, read_lsn)))); + } reset_tmp_lsn_info_array(lsn_info); get_lsn_info_for_read(buf_tag, block_meta_info->base_page_info_list.prev, lsn_info, read_lsn); UnlockReleaseBuffer(buf); + + if (lsn_info->lsn_num == 0 && XLogRecPtrIsInvalid(lsn_info->base_page_lsn)) { + ereport(ERROR, ((errmsg("cannot find valid lsn info %u/%u/%u %d %u read lsn %lu, min lsn %lu", + buf_tag.rnode.spcNode, buf_tag.rnode.dbNode, buf_tag.rnode.relNode, buf_tag.forkNum, + buf_tag.blockNum, read_lsn, block_meta_info->min_lsn)))); + } return true; } @@ -227,73 +275,12 @@ void remove_one_block_info_file(const RelFileNode rnode) * recycle all relation files when drop db occurs. * db_id: database oid. */ -void remove_block_meta_info_files_of_db(Oid db_oid, Oid rel_oid) +void remove_block_meta_info_files_of_db(Oid db_oid) { - char pathbuf[EXRTO_FILE_PATH_LEN]; - char **filenames; - char **filename; - struct stat statbuf; - /* get block info file directory */ - char exrto_block_info_dir[EXRTO_FILE_PATH_LEN] = {0}; - int rc = snprintf_s(exrto_block_info_dir, EXRTO_FILE_PATH_LEN, EXRTO_FILE_PATH_LEN - 1, "%s/%s", EXRTO_FILE_DIR, - EXRTO_FILE_SUB_DIR[BLOCK_INFO_META]); - securec_check_ss(rc, "", ""); - /* get all files' name from block meta file directory */ - filenames = pgfnames(exrto_block_info_dir); - if (filenames == NULL) { - return; - } char target_prefix[EXRTO_FILE_PATH_LEN] = {0}; - if (rel_oid != InvalidOid) { - rc = sprintf_s(target_prefix, EXRTO_FILE_PATH_LEN, "%u_%u_", db_oid, rel_oid); - } else { - rc = sprintf_s(target_prefix, EXRTO_FILE_PATH_LEN, "%u_", db_oid); - } + errno_t rc = sprintf_s(target_prefix, EXRTO_FILE_PATH_LEN, "%u_", db_oid); securec_check_ss(rc, "", ""); - /* use the prefix name to match up files we want to delete */ - size_t prefix_len = strlen(target_prefix); - for (filename = filenames; *filename != NULL; filename++) { - char *fname = *filename; - size_t fname_len = strlen(fname); - /* - * the length of prefix is less than the length of file name and must be the same under the same prefix_len - */ - if (prefix_len >= fname_len || strncmp(target_prefix, fname, prefix_len) != 0) { - continue; - } - rc = - snprintf_s(pathbuf, EXRTO_FILE_PATH_LEN, EXRTO_FILE_PATH_LEN - 1, "%s/%s", exrto_block_info_dir, *filename); - securec_check_ss(rc, "", ""); - /* may be can be some error */ - if (lstat(pathbuf, &statbuf) != 0) { - if (errno != ENOENT) { -#ifndef FRONTEND - ereport(WARNING, (errmsg("could not stat file or directory \"%s\" \n", pathbuf))); -#else - fprintf(stderr, _("could not stat file or directory \"%s\": %s\n"), pathbuf, gs_strerror(errno)); -#endif - } - continue; - } - /* if the file is a directory, don't touch it */ - if (S_ISDIR(statbuf.st_mode)) { - /* skip dir */ - continue; - } - /* delete this file we found */ - if (unlink(pathbuf) != 0) { - if (errno != ENOENT) { -#ifndef FRONTEND - ereport(WARNING, (errmsg("could not remove file or directory \"%s\" ", pathbuf))); -#else - fprintf(stderr, _("could not remove file or directory \"%s\": %s\n"), pathbuf, gs_strerror(errno)); -#endif - } - } - } - pgfnames_cleanup(filenames); - return; + exrto_unlink_file_with_prefix(target_prefix, BLOCK_INFO_META); } } // namespace extreme_rto_standby_read - diff --git a/src/gausskernel/storage/access/redo/standby_read/lsn_info_double_list.cpp b/src/gausskernel/storage/access/redo/standby_read/lsn_info_double_list.cpp index bffb2be17..0c5827af5 100644 --- a/src/gausskernel/storage/access/redo/standby_read/lsn_info_double_list.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/lsn_info_double_list.cpp @@ -63,7 +63,6 @@ void info_list_modify_old_tail(StandbyReadMetaInfo *meta_info, LsnInfoPosition o base_page_info->base_page_list.next = insert_pos; base_page_info->next_base_page_lsn = current_page_lsn; Assert(is_lsn_info_node_valid(base_page_info->lsn_info_node.flags)); - Assert(XLByteLT(base_page_info->cur_page_lsn, current_page_lsn)); } standby_read_meta_page_set_lsn(page, next_lsn); diff --git a/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp b/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp index 765538955..622774215 100644 --- a/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp @@ -16,7 +16,7 @@ * lsn_info_proc.cpp * * IDENTIFICATION - * src/gausskernel/storage/recovery/parallel/blocklevel/standby_read/lsn_info_proc.cpp + * src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp * * ------------------------------------------------------------------------- */ @@ -98,9 +98,13 @@ Page get_lsn_info_page(uint32 batch_id, uint32 worker_id, LsnInfoPosition positi batch_id, worker_id))); return NULL; } - + +#ifdef ENABLE_UT + page = get_page_from_buffer(*buffer); +#else page = BufferGetPage(*buffer); - if (!is_lsn_info_page_valid((LsnInfoPageHeader*)page)) { +#endif + if (!is_lsn_info_page_valid((LsnInfoPageHeader *)page)) { if (mode == RBM_NORMAL) { ReleaseBuffer(*buffer); *buffer = InvalidBuffer; @@ -109,7 +113,7 @@ Page get_lsn_info_page(uint32 batch_id, uint32 worker_id, LsnInfoPosition positi /* make sure to make buffer dirty outside */ lsn_info_page_init(page); } - + return page; } @@ -312,17 +316,16 @@ void get_lsn_info_for_read(const BufferTag& buf_tag, LsnInfoPosition latest_lsn_ /* get batch id and page redo worker id */ extreme_rto::RedoItemTag redo_item_tag; - const uint32 worker_num_per_mng = extreme_rto::get_page_redo_worker_num_per_manager(); INIT_REDO_ITEM_TAG(redo_item_tag, buf_tag.rnode, buf_tag.forkNum, buf_tag.blockNum); /* batch id and worker id start from 1 when reading a page */ - batch_id = extreme_rto::GetSlotId(buf_tag.rnode, 0, 0, extreme_rto::get_batch_redo_num()) + 1; - worker_id = extreme_rto::GetWorkerId(&redo_item_tag, worker_num_per_mng) + 1; + batch_id = extreme_rto::GetSlotId(buf_tag.rnode, 0, 0, (uint32)extreme_rto::get_batch_redo_num()) + 1; + worker_id = extreme_rto::GetWorkerId(&redo_item_tag, extreme_rto::get_page_redo_worker_num_per_manager()) + 1; /* find fisrt base page whose lsn less than read lsn form tail to head */ do { /* reach the end of the list */ if (INFO_POSITION_IS_INVALID(latest_lsn_base_page_pos)) { - ereport(ERROR, ( + ereport(PANIC, ( errmsg("can not find base page, block is %u/%u/%u %d %u, batch_id: %u, redo_worker_id: %u", buf_tag.rnode.spcNode, buf_tag.rnode.dbNode, buf_tag.rnode.relNode, buf_tag.forkNum, buf_tag.blockNum, batch_id, worker_id))); @@ -346,7 +349,7 @@ void get_lsn_info_for_read(const BufferTag& buf_tag, LsnInfoPosition latest_lsn_ Assert(is_base_page_type(base_page_info->lsn_info_node.type)); /* If we find the desired page, keep it locked */ - if (XLByteLT(page_lsn, read_lsn)) { + if (XLByteLE(page_lsn, read_lsn)) { break; } UnlockReleaseBuffer(buffer); @@ -422,7 +425,7 @@ void set_base_page_map_bit(Page page, uint32 base_page_loc) * base_page_loc must be an integer multiple of LSN_INFO_HEAD_SIZE */ check_base_page_loc_valid(base_page_loc); - + LsnInfoPageHeader *page_header = (LsnInfoPageHeader *)page; uint8 *base_page_map = page_header->base_page_map; uint32 which_bit = base_page_loc / LSN_INFO_NODE_SIZE; @@ -469,25 +472,26 @@ void recycle_one_lsn_info_list(const BufferTag& buf_tag, LsnInfoPosition page_in { /* get batch id and page redo worker id */ extreme_rto::RedoItemTag redo_item_tag; - const uint32 worker_num_per_mng = extreme_rto::get_page_redo_worker_num_per_manager(); INIT_REDO_ITEM_TAG(redo_item_tag, buf_tag.rnode, buf_tag.forkNum, buf_tag.blockNum); + + const uint32 worker_num_per_mng = extreme_rto::get_page_redo_worker_num_per_manager(); /* batch id and worker id start from 1 when reading a page */ - uint32 batch_id = extreme_rto::GetSlotId(buf_tag.rnode, 0, 0, extreme_rto::get_batch_redo_num()) + 1; + uint32 batch_id = extreme_rto::GetSlotId(buf_tag.rnode, 0, 0, (uint32)extreme_rto::get_batch_redo_num()) + 1; uint32 worker_id = extreme_rto::GetWorkerId(&redo_item_tag, worker_num_per_mng) + 1; while (INFO_POSITION_IS_VALID(page_info_pos)) { Buffer buffer = InvalidBuffer; Page page = get_lsn_info_page(batch_id, worker_id, page_info_pos, RBM_NORMAL, &buffer); if (unlikely(page == NULL || buffer == InvalidBuffer)) { - ereport(PANIC, (errmsg(EXRTOFORMAT("get_lsn_info_page failed, batch_id: %u, redo_id: %u, pos: %lu"), - batch_id, worker_id, page_info_pos))); + ereport(PANIC, (errmsg(EXRTOFORMAT("get_lsn_info_page failed, batch_id: %u, redo_id: %u, pos: %lu"), + batch_id, worker_id, page_info_pos))); } LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); - + uint32 offset = lsn_info_postion_to_offset(page_info_pos); BasePageInfo base_page_info = (BasePageInfo)(page + offset); Assert(is_base_page_type(base_page_info->lsn_info_node.type)); - + *min_page_info_pos = page_info_pos; *min_lsn = base_page_info->cur_page_lsn; @@ -508,7 +512,11 @@ void recycle_one_lsn_info_list(const BufferTag& buf_tag, LsnInfoPosition page_in void invalid_base_page_list(StandbyReadMetaInfo *meta_info, Buffer buffer, uint32 offset) { LsnInfoPosition page_info_pos; +#ifdef ENABLE_UT + Page page = get_page_from_buffer(buffer); +#else Page page = BufferGetPage(buffer); +#endif BasePageInfo base_page_info = (BasePageInfo)(page + offset); /* set invalid flags */ LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); @@ -522,12 +530,12 @@ void invalid_base_page_list(StandbyReadMetaInfo *meta_info, Buffer buffer, uint3 while (INFO_POSITION_IS_VALID(page_info_pos)) { page = get_lsn_info_page(batch_id, worker_id, page_info_pos, RBM_NORMAL, &buffer); if (unlikely(page == NULL || buffer == InvalidBuffer)) { - ereport(PANIC, (errmsg(EXRTOFORMAT("get_lsn_info_page failed, batch_id: %u, redo_id: %u, pos: %lu"), - batch_id, worker_id, page_info_pos))); + ereport(PANIC, (errmsg(EXRTOFORMAT("get_lsn_info_page failed, batch_id: %u, redo_id: %u, pos: %lu"), + batch_id, worker_id, page_info_pos))); } offset = lsn_info_postion_to_offset(page_info_pos); base_page_info = (BasePageInfo)(page + offset); - + /* unset valid flags */ LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); base_page_info->lsn_info_node.flags &= ~LSN_INFO_NODE_VALID_FLAG; @@ -558,10 +566,10 @@ bool recycle_one_lsn_info_page(StandbyReadMetaInfo *meta_info, XLogRecPtr recycl LsnInfoPosition recycle_pos = meta_info->lsn_table_recyle_position; Page page = get_lsn_info_page(batch_id, worker_id, recycle_pos, RBM_NORMAL, &buffer); if (unlikely(page == NULL || buffer == InvalidBuffer)) { - ereport(PANIC, (errmsg(EXRTOFORMAT("get_lsn_info_page failed, batch_id: %u, redo_id: %u, pos: %lu"), - batch_id, worker_id, recycle_pos))); + ereport(PANIC, (errmsg(EXRTOFORMAT("get_lsn_info_page failed, batch_id: %u, redo_id: %u, pos: %lu"), batch_id, + worker_id, recycle_pos))); } - + bool buffer_is_locked = false; /* skip page header */ for (uint32 bit = 1; bit < BASE_PAGE_MAP_SIZE * BYTE_BITS; bit++) { @@ -627,7 +635,10 @@ void standby_read_recyle_per_workers(StandbyReadMetaInfo *meta_info, XLogRecPtr Assert(meta_info->redo_id > 0); bool recycle_next_page = true; BasePagePosition base_page_position = meta_info->base_page_recyle_position; - + uint64 last_base_page_recyle_segno = meta_info->base_page_recyle_position / EXRTO_BASE_PAGE_FILE_MAXSIZE; + uint64 last_lsn_table_recyle_segno = meta_info->lsn_table_recyle_position / EXRTO_LSN_INFO_FILE_MAXSIZE; + uint64 cur_base_page_recyle_segno, cur_lsn_table_recyle_segno; + while (meta_info->lsn_table_recyle_position + BLCKSZ < meta_info->lsn_table_next_position) { recycle_next_page = recycle_one_lsn_info_page(meta_info, recycle_lsn, &base_page_position); if (!recycle_next_page) { @@ -643,8 +654,18 @@ void standby_read_recyle_per_workers(StandbyReadMetaInfo *meta_info, XLogRecPtr Assert(meta_info->base_page_recyle_position % BLCKSZ == 0); Assert(meta_info->base_page_recyle_position <= meta_info->base_page_next_position); - recycle_lsn_info_file(meta_info->batch_id, meta_info->redo_id, meta_info->lsn_table_recyle_position); - recycle_base_page_file(meta_info->batch_id, meta_info->redo_id, meta_info->base_page_recyle_position); + cur_base_page_recyle_segno = meta_info->base_page_recyle_position / EXRTO_BASE_PAGE_FILE_MAXSIZE; + cur_lsn_table_recyle_segno = meta_info->lsn_table_recyle_position / EXRTO_LSN_INFO_FILE_MAXSIZE; + if (cur_base_page_recyle_segno > last_base_page_recyle_segno || + cur_lsn_table_recyle_segno > last_lsn_table_recyle_segno) { + buffer_drop_exrto_standby_read_buffers(meta_info); + } + if (cur_lsn_table_recyle_segno > last_lsn_table_recyle_segno) { + recycle_lsn_info_file(meta_info->batch_id, meta_info->redo_id, meta_info->lsn_table_recyle_position); + } + if (cur_base_page_recyle_segno > last_base_page_recyle_segno) { + recycle_base_page_file(meta_info->batch_id, meta_info->redo_id, meta_info->base_page_recyle_position); + } } } // namespace extreme_rto_standby_read diff --git a/src/gausskernel/storage/access/redo/standby_read/standby_read_delay_ddl.cpp b/src/gausskernel/storage/access/redo/standby_read/standby_read_delay_ddl.cpp new file mode 100644 index 000000000..c9adb75a6 --- /dev/null +++ b/src/gausskernel/storage/access/redo/standby_read/standby_read_delay_ddl.cpp @@ -0,0 +1,448 @@ +/* + * Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * standby_read_delay_ddl.cpp + * + * IDENTIFICATION + * src/gausskernel/storage/access/redo/standby_read/standby_read_delay_ddl.cpp + * + * ------------------------------------------------------------------------- + */ + +#include +#include +#include +#include "access/extreme_rto/standby_read.h" +#include "access/extreme_rto/standby_read/standby_read_delay_ddl.h" +#include "access/extreme_rto/standby_read/block_info_meta.h" +#include "access/multi_redo_api.h" +#include "commands/dbcommands.h" + +#define DELAY_DDL_FILE_DIR "delay_ddl" +#define DELAY_DDL_FILE_NAME "delay_ddl/delay_delete_info_file" + +typedef enum { + DROP_DB_TYPE = 1, + DROP_TABLE_TYPE, +} DropDdlType; + +const static uint32 MAX_NUM_PER_FILE = 0x10000; + +typedef struct { + uint8 type; + uint8 len; + uint16 resvd1; + uint32 resvd2; + ColFileNode node_info; + XLogRecPtr lsn; + pg_crc32 crc; +} DelayDdlInfo; + +void init_delay_ddl_file() +{ + if (isDirExist(DELAY_DDL_FILE_DIR)) { + return; + } + + if (!IS_EXRTO_READ) { + return; + } + + if (mkdir(DELAY_DDL_FILE_DIR, S_IRWXU) < 0 && errno != EEXIST) { + ereport(ERROR, + (errcode_for_file_access(), errmsg("could not create directory \"%s\": %m", DELAY_DDL_FILE_DIR))); + } +} + +int read_delay_ddl_info(int fd, void* buf, size_t size, off_t off) +{ + int count = 0; +RETRY: + errno = 0; + int return_code = pread(fd, buf, size, off); + if (return_code < 0) { + /* OK to retry if interrupted */ + if (errno == EINTR) { + goto RETRY; + } + + if (errno == EIO) { + if (count < EIO_RETRY_TIMES) { + count++; + ereport(WARNING, + (errmsg("delete_by_lsn: failed (read len %lu, offset %ld), retry:Input/Output ERROR", size, off))); + goto RETRY; + } + } + } + + return return_code; +} + +bool write_delay_ddl_info(char* file_path, void* buf, size_t size, off_t off) +{ + int fd = BasicOpenFile(file_path, O_CREAT | O_RDWR | PG_BINARY, S_IRUSR | S_IWUSR); + if (fd < 0) { + ereport(WARNING, + (errcode_for_file_access(), errmsg("write_delay_ddl_info:could not open file %s : %m", file_path))); + return false; + } + int count = 0; +RETRY: + errno = 0; + int return_code = pwrite(fd, buf, size, off); + if (return_code != (int)size && errno == 0) { + errno = ENOSPC; + } + + /* OK to retry if interrupted */ + if (errno == EINTR) { + goto RETRY; + } + if (errno == EIO) { + if (count < EIO_RETRY_TIMES) { + count++; + ereport(WARNING, + (errmsg("write_delay_ddl_info: write %s failed, then retry: Input/Output ERROR", file_path))); + goto RETRY; + } + } + + if (return_code != (int)size) { + ereport(WARNING, (errcode_for_file_access(), + errmsg("write_delay_ddl_info:write maybe failed %s ,write %d, need %lu, offset %ld: %m", + file_path, return_code, size, off))); + close(fd); + return false; + } + + if (fsync(fd) != 0) { + ereport(WARNING, + (errcode_for_file_access(), errmsg("write_delay_ddl_info:could not fsync file %s: %m", file_path))); + close(fd); + return false; + } + + close(fd); + return true; +} + + +static void enter_state(uint32 *stat) +{ + uint32 expected = 0; + while (!pg_atomic_compare_exchange_u32(stat, &expected, 1)) { + expected = 0; + RedoInterruptCallBack(); + } +} + +static void exit_state(uint32 *stat) +{ + (void)pg_atomic_sub_fetch_u32(stat, 1); +} + +void update_delay_ddl_db(Oid db_id, Oid tablespace_id, XLogRecPtr lsn) +{ + StandbyReadDelayDdlState *stat = &g_instance.comm_cxt.predo_cxt.standby_read_delay_ddl_stat; + enter_state(&stat->insert_stat); + uint64 insert_start = pg_atomic_read_u64(&stat->next_index_can_insert); + + char path[MAXPGPATH]; + errno_t errorno = snprintf_s(path, MAXPGPATH, MAXPGPATH - 1, DELAY_DDL_FILE_NAME "_%08X_%lX", + t_thrd.shemem_ptr_cxt.ControlFile->timeline, insert_start / MAX_NUM_PER_FILE); + securec_check_ss(errorno, "", ""); + + off_t off_set = (off_t)(insert_start % MAX_NUM_PER_FILE * sizeof(DelayDdlInfo)); + + DelayDdlInfo tmp_info = { + .type = DROP_DB_TYPE, + .len = sizeof(DelayDdlInfo), + .resvd1 = 0, + .resvd2 = 0, + }; + + tmp_info.node_info.filenode.dbNode = db_id; + tmp_info.node_info.filenode.spcNode = tablespace_id; + tmp_info.lsn = lsn; + INIT_CRC32C(tmp_info.crc); + COMP_CRC32C(tmp_info.crc, (char*)&tmp_info, offsetof(DelayDdlInfo, crc)); + FIN_CRC32C(tmp_info.crc); + + if (write_delay_ddl_info(path, &tmp_info, sizeof(DelayDdlInfo), off_set)) { + pg_atomic_write_u64(&stat->next_index_can_insert, insert_start + 1); + } + exit_state(&stat->insert_stat); +} + +void update_delay_ddl_files(ColFileNode* xnodes, int nrels, XLogRecPtr lsn) +{ + DelayDdlInfo* info_list = (DelayDdlInfo*)palloc(sizeof(DelayDdlInfo) * (uint32)nrels); + for (int i = 0; i < nrels; ++i) { + info_list[i].type = DROP_TABLE_TYPE; + info_list[i].len = sizeof(DelayDdlInfo); + info_list[i].resvd1 = 0; + info_list[i].resvd2 = 0; + info_list[i].node_info = xnodes[i]; + info_list[i].lsn = lsn; + INIT_CRC32C(info_list[i].crc); + COMP_CRC32C(info_list[i].crc, (char*)&info_list[i], offsetof(DelayDdlInfo, crc)); + FIN_CRC32C(info_list[i].crc); + } + uint32 remains = (uint32)nrels; + StandbyReadDelayDdlState *stat = &g_instance.comm_cxt.predo_cxt.standby_read_delay_ddl_stat; + enter_state(&stat->insert_stat); + uint64 insert_start = pg_atomic_read_u64(&stat->next_index_can_insert); + + DelayDdlInfo *start = info_list; + while (remains > 0) { + uint32 left_size = MAX_NUM_PER_FILE - insert_start % MAX_NUM_PER_FILE; + uint32 copys = remains; + if (remains > left_size) { + copys = (uint32)left_size; + } + + char path[MAXPGPATH]; + errno_t errorno = snprintf_s(path, MAXPGPATH, MAXPGPATH - 1, DELAY_DDL_FILE_NAME "_%08X_%lX", + t_thrd.shemem_ptr_cxt.ControlFile->timeline, insert_start / MAX_NUM_PER_FILE); + securec_check_ss(errorno, "", ""); + + off_t off_set = (off_t)(insert_start % MAX_NUM_PER_FILE * sizeof(DelayDdlInfo)); + + if (write_delay_ddl_info(path, start, copys * sizeof(DelayDdlInfo), off_set)) { + remains -= copys; + start += copys; + insert_start += copys; + } else { + break; + } + RedoInterruptCallBack(); + } + pfree(info_list); + + pg_atomic_write_u64(&stat->next_index_can_insert, insert_start); + exit_state(&stat->insert_stat); +} + +void do_delay_ddl(DelayDdlInfo* info) +{ + pg_crc32c crc_check; + INIT_CRC32C(crc_check); + COMP_CRC32C(crc_check, (char*)info, offsetof(DelayDdlInfo, crc)); + FIN_CRC32C(crc_check); + + if (!EQ_CRC32C(crc_check, info->crc)) { + ereport(WARNING, (errcode_for_file_access(), + errmsg("delay ddl ,crc(%u:%u) check error, maybe is type:%u, info %u/%u/%u lsn:%lu", crc_check, info->crc, + (uint32)info->type, info->node_info.filenode.spcNode, info->node_info.filenode.dbNode, + info->node_info.filenode.relNode, info->lsn))); + return; + } + + if (info->type == DROP_TABLE_TYPE) { + unlink_relfiles(&info->node_info, 1); + xact_redo_log_drop_segs(&info->node_info, 1, info->lsn); + } else if (info->type == DROP_DB_TYPE) { + do_db_drop(info->node_info.filenode.dbNode, info->node_info.filenode.spcNode); + } else { + ereport(WARNING, (errcode_for_file_access(), + errmsg("delay ddl ,type error, maybe is type:%u, info %u/%u/%u lsn:%lu", (uint32)info->type, + info->node_info.filenode.spcNode, info->node_info.filenode.dbNode, info->node_info.filenode.relNode, + info->lsn))); + } +} + +void delete_by_lsn(XLogRecPtr lsn) +{ + StandbyReadDelayDdlState *stat = &g_instance.comm_cxt.predo_cxt.standby_read_delay_ddl_stat; + enter_state(&stat->delete_stat); + + uint64 next_delete = pg_atomic_read_u64(&stat->next_index_need_unlink); + uint64 next_insert = pg_atomic_read_u64(&stat->next_index_can_insert); + DelayDdlInfo* info_list = (DelayDdlInfo*)palloc(sizeof(DelayDdlInfo) * MAX_NUM_PER_FILE); + bool go_on_delete = true; + uint64 deleted_total = 0; + while (next_delete < next_insert && go_on_delete) { + uint32 cur_deleted = MAX_NUM_PER_FILE; + + /* same file */ + if ((next_delete / MAX_NUM_PER_FILE) == (next_insert / MAX_NUM_PER_FILE)) { + cur_deleted = next_insert - next_delete; + } else { /* different file */ + cur_deleted = MAX_NUM_PER_FILE - next_delete % MAX_NUM_PER_FILE; + } + + uint64 offset = next_delete % MAX_NUM_PER_FILE * sizeof(DelayDdlInfo); + + char path[MAXPGPATH]; + errno_t errorno = snprintf_s(path, MAXPGPATH, MAXPGPATH - 1, DELAY_DDL_FILE_NAME "_%08X_%lX", + t_thrd.shemem_ptr_cxt.ControlFile->timeline, next_delete / MAX_NUM_PER_FILE); + securec_check_ss(errorno, "", ""); + + int fd = BasicOpenFile(path, O_RDONLY | PG_BINARY, S_IRUSR | S_IWUSR); + if (fd < 0) { + return; + } + int count = read_delay_ddl_info(fd, info_list, cur_deleted * sizeof(DelayDdlInfo), (off_t)offset); + close(fd); + if (count <= 0) { + ereport(WARNING, + (errmsg("delete_by_lsn: file %s nothing deleted_total", path))); + go_on_delete = false; + continue; + } + if ((uint32)count / sizeof(DelayDdlInfo) < cur_deleted) { + cur_deleted = (uint32)count / sizeof(DelayDdlInfo); + ereport(WARNING, + (errmsg("delete_by_lsn: the info in file %s is less than expected, cur_deleted: %u, next_delete: " + "%lu, next_insert: %lu", + path, cur_deleted, next_delete, next_insert))); + } + + for (uint32 i = 0; i < cur_deleted; ++i) { + if (info_list[i].lsn <= lsn) { + do_delay_ddl(&info_list[i]); + } else { + cur_deleted = i; + go_on_delete = false; + break; + } + RedoInterruptCallBack(); + } + + next_delete += cur_deleted; + deleted_total += cur_deleted; + if (next_delete % MAX_NUM_PER_FILE == 0) { + (void)unlink(path); + } + RedoInterruptCallBack(); + } + ereport(LOG, (errmsg("delete_by_lsn: unlink files number: %lu", deleted_total))); + pfree(info_list); + pg_atomic_write_u64(&stat->next_index_need_unlink, next_delete); + exit_state(&stat->delete_stat); +} + +void delete_by_table_space(Oid tablespace_id) +{ + StandbyReadDelayDdlState *stat = &g_instance.comm_cxt.predo_cxt.standby_read_delay_ddl_stat; + enter_state(&stat->delete_stat); + uint64 next_delete = pg_atomic_read_u64(&stat->next_index_need_unlink); + uint64 next_insert = pg_atomic_read_u64(&stat->next_index_can_insert); + + DelayDdlInfo* info_list = (DelayDdlInfo*)palloc0(sizeof(DelayDdlInfo) * MAX_NUM_PER_FILE); + while (next_delete < next_insert) { + uint32 copys = MAX_NUM_PER_FILE; + + // same file + if ((next_delete / MAX_NUM_PER_FILE) == (next_insert / MAX_NUM_PER_FILE)) { + copys = next_insert - next_delete; + } else { /* different file */ + copys = MAX_NUM_PER_FILE - next_delete % MAX_NUM_PER_FILE; + } + + uint64 offset = next_delete % MAX_NUM_PER_FILE * sizeof(DelayDdlInfo); + char path[MAXPGPATH]; + errno_t errorno = snprintf_s(path, MAXPGPATH, MAXPGPATH - 1, DELAY_DDL_FILE_NAME "_%08X_%lX", + t_thrd.shemem_ptr_cxt.ControlFile->timeline, next_delete / MAX_NUM_PER_FILE); + securec_check_ss(errorno, "", ""); + + int fd = BasicOpenFile(path, O_RDONLY | PG_BINARY, S_IRUSR | S_IWUSR); + if (fd < 0) { + ereport(WARNING, + (errmsg("delete_by_table_space: file %s could not open:%m", path))); + return; + } + + int count = read_delay_ddl_info(fd, info_list, copys * sizeof(DelayDdlInfo), (off_t)offset); + if (count <= 0) { + ereport(WARNING, + (errmsg("delete_by_table_space: file %s nothing deleted", path))); + return; + } + close(fd); + + if ((uint32)count / sizeof(DelayDdlInfo) < copys) { + copys = (uint32)count / sizeof(DelayDdlInfo); + ereport( + WARNING, + (errmsg("delete_by_table_space: the info in file %s is less than expected, copys: %u, next_delete: " + "%lu, next_insert: %lu", + path, copys, next_delete, next_insert))); + } + + for (uint32 i = 0; i < copys; ++i) { + if (info_list[i].node_info.filenode.spcNode == tablespace_id) { + do_delay_ddl(&info_list[i]); + } + RedoInterruptCallBack(); + } + next_delete += copys; + RedoInterruptCallBack(); + } + pfree(info_list); + exit_state(&stat->delete_stat); +} + +void do_all_old_delay_ddl() +{ + DIR* file_dir = AllocateDir(DELAY_DDL_FILE_DIR); + struct dirent* file_dirent = NULL; + uint32 timeline = 0; + uint64 segment = 0; + while ((file_dirent = ReadDir(file_dir, DELAY_DDL_FILE_DIR)) != NULL) { + int nmatch = sscanf_s(file_dirent->d_name, "delay_delete_info_file_%08X_%lX", &timeline, &segment); + if (nmatch != 2) { + continue; + } + if (timeline >= t_thrd.shemem_ptr_cxt.ControlFile->timeline && RecoveryInProgress()) { + continue; + } + DelayDdlInfo* info_list = (DelayDdlInfo*)palloc0(sizeof(DelayDdlInfo) * MAX_NUM_PER_FILE); + char path[MAXPGPATH]; + errno_t errorno = snprintf_s(path, MAXPGPATH, MAXPGPATH - 1, DELAY_DDL_FILE_DIR "/%s", file_dirent->d_name); + securec_check_ss(errorno, "", ""); + + int fd = BasicOpenFile(path, O_RDONLY | PG_BINARY, S_IRUSR | S_IWUSR); + if (fd < 0) { + ereport(WARNING, + (errcode_for_file_access(), errmsg("open_delay_store_file:could not open file \"%s\": %m", path))); + } + + int count = read_delay_ddl_info(fd, info_list, MAX_NUM_PER_FILE * sizeof(DelayDdlInfo), 0); + close(fd); + if (count <= 0) { + return; + } + + for (uint32 i = 0; i < (uint32)(count / sizeof(DelayDdlInfo)); ++i) { + if (XLByteLE(info_list[i].lsn, t_thrd.shemem_ptr_cxt.ControlFile->checkPointCopy.redo)) { + do_delay_ddl(&info_list[i]); + } + RedoInterruptCallBack(); + } + + (void)unlink(path); + pfree(info_list); + RedoInterruptCallBack(); + } + + if (!RecoveryInProgress()) { + g_instance.comm_cxt.predo_cxt.standby_read_delay_ddl_stat.next_index_can_insert = 0; + g_instance.comm_cxt.predo_cxt.standby_read_delay_ddl_stat.next_index_need_unlink = 0; + g_instance.comm_cxt.predo_cxt.standby_read_delay_ddl_stat.delete_stat = 0; + g_instance.comm_cxt.predo_cxt.standby_read_delay_ddl_stat.insert_stat = 0; + } +} \ No newline at end of file diff --git a/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp b/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp index f64492a46..145fa84a1 100644 --- a/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp @@ -21,43 +21,54 @@ * ------------------------------------------------------------------------- */ +#include #include #include "access/extreme_rto/page_redo.h" #include "access/extreme_rto/standby_read/block_info_meta.h" #include "access/extreme_rto/standby_read/lsn_info_meta.h" +#include "access/extreme_rto/standby_read/standby_read_delay_ddl.h" #include "access/multi_redo_api.h" #include "pgstat.h" #include "storage/smgr/relfilenode.h" #include "storage/buf/buf_internals.h" #include "storage/buf/bufmgr.h" +#include "../../../page/pageparse.h" #include "storage/smgr/segment.h" #include "utils/rel.h" #include "utils/palloc.h" #include "access/extreme_rto/dispatcher.h" #include "funcapi.h" + const char* EXRTO_BASE_PAGE_SUB_DIR = "base_page"; const char* EXRTO_LSN_INFO_SUB_DIR = "lsn_info_meta"; const char* EXRTO_BLOCK_INFO_SUB_DIR = "block_info_meta"; const char* EXRTO_FILE_SUB_DIR[] = { EXRTO_BASE_PAGE_SUB_DIR, EXRTO_LSN_INFO_SUB_DIR, EXRTO_BLOCK_INFO_SUB_DIR}; const uint32 EXRTO_FILE_PATH_LEN = 1024; +const uint32 XID_THIRTY_TWO = 32; -void make_standby_read_node(XLogRecPtr read_lsn, RelFileNode& read_node) +void make_standby_read_node(XLogRecPtr read_lsn, RelFileNode &read_node, bool is_start_lsn) { read_node.spcNode = (Oid)(read_lsn >> 32); read_node.dbNode = (Oid)(read_lsn); - read_node.relNode = InvalidOid; // make sure it can be InvalidOid or not + read_node.relNode = InvalidOid; // make sure it can be InvalidOid or not read_node.opt = 0; - read_node.bucketNode = InvalidBktId; + if (is_start_lsn) { + /* means read_lsn is the start ptr of xlog */ + read_node.bucketNode = ExrtoReadStartLSNBktId; + } else { + /* means read_lsn is the end ptr of xlog */ + read_node.bucketNode = ExrtoReadEndLSNBktId; + } } -BufferDesc* alloc_standby_read_buf( - const BufferTag& buf_tag, BufferAccessStrategy strategy, bool& found, XLogRecPtr read_lsn) +BufferDesc *alloc_standby_read_buf(const BufferTag &buf_tag, BufferAccessStrategy strategy, bool &found, + XLogRecPtr read_lsn, bool is_start_lsn) { RelFileNode read_node; - make_standby_read_node(read_lsn, read_node); - BufferDesc* buf_desc = BufferAlloc(read_node, 0, buf_tag.forkNum, buf_tag.blockNum, strategy, &found, NULL); + make_standby_read_node(read_lsn, read_node, is_start_lsn); + BufferDesc *buf_desc = BufferAlloc(read_node, 0, buf_tag.forkNum, buf_tag.blockNum, strategy, &found, NULL); return buf_desc; } @@ -87,7 +98,7 @@ Buffer get_newest_page_for_read(Relation reln, ForkNumber fork_num, BlockNumber .blockNum = block_num, }; ResourceOwnerEnlargeBuffers(t_thrd.utils_cxt.CurrentResourceOwner); - BufferDesc* buf_desc = alloc_standby_read_buf(buf_tag, strategy, hit, page_lsn); + BufferDesc* buf_desc = alloc_standby_read_buf(buf_tag, strategy, hit, page_lsn, false); if (hit) { UnlockReleaseBuffer(newest_buf); @@ -125,10 +136,12 @@ Buffer standby_read_buf( .forkNum = fork_num, .blockNum = block_num, }; - XLogRecPtr read_lsn = t_thrd.proc->exrto_read_lsn; - if (read_lsn == InvalidXLogRecPtr) { - Assert(IsSystemRelation(reln)); - read_lsn = MAX_XLOG_REC_PTR; + + XLogRecPtr read_lsn = MAX_XLOG_REC_PTR; + if (u_sess->utils_cxt.CurrentSnapshot != NULL && XLogRecPtrIsValid(u_sess->utils_cxt.CurrentSnapshot->read_lsn)) { + read_lsn = u_sess->utils_cxt.CurrentSnapshot->read_lsn; + } else if (XLogRecPtrIsValid(t_thrd.proc->exrto_read_lsn)) { + read_lsn = t_thrd.proc->exrto_read_lsn; } Buffer read_buf = get_newest_page_for_read(reln, fork_num, block_num, mode, strategy, read_lsn); @@ -142,17 +155,24 @@ Buffer standby_read_buf( StandbyReadLsnInfoArray *lsn_info = &t_thrd.exrto_recycle_cxt.lsn_info; bool result = extreme_rto_standby_read::get_page_lsn_info(buf_tag, strategy, read_lsn, lsn_info); if (!result) { - ereport(ERROR, - (errcode(ERRCODE_INTERNAL_ERROR), - (errmsg("standby_read_buf couldnot found buf %u/%u/%u %d %u read lsn %lu", buf_tag.rnode.spcNode, - buf_tag.rnode.dbNode, buf_tag.rnode.relNode, buf_tag.forkNum, buf_tag.blockNum, read_lsn)))); + ereport( + ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + (errmsg("standby_read_buf couldnot found buf %u/%u/%u %d %u read lsn %08X/%08X current_time: %ld " + "gen_snaptime:%ld thread_read_lsn:%08X/%08X", + buf_tag.rnode.spcNode, buf_tag.rnode.dbNode, buf_tag.rnode.relNode, buf_tag.forkNum, + buf_tag.blockNum, (uint32)(read_lsn >> XID_THIRTY_TWO), (uint32)read_lsn, GetCurrentTimestamp(), + g_instance.comm_cxt.predo_cxt.exrto_snapshot->gen_snap_time, + (uint32)(t_thrd.proc->exrto_read_lsn >> XID_THIRTY_TWO), (uint32)t_thrd.proc->exrto_read_lsn)))); return InvalidBuffer; } // read lsn info XLogRecPtr expected_lsn = InvalidXLogRecPtr; + bool is_start_lsn = true; if (lsn_info->lsn_num == 0) { expected_lsn = lsn_info->base_page_lsn; + is_start_lsn = false; } else { Assert(lsn_info->lsn_array[lsn_info->lsn_num - 1] > 0); Assert(lsn_info->lsn_array[lsn_info->lsn_num - 1] < read_lsn); @@ -160,7 +180,7 @@ Buffer standby_read_buf( expected_lsn = lsn_info->lsn_array[lsn_info->lsn_num - 1]; } - BufferDesc* buf_desc = alloc_standby_read_buf(buf_tag, strategy, hit, expected_lsn); + BufferDesc* buf_desc = alloc_standby_read_buf(buf_tag, strategy, hit, expected_lsn, is_start_lsn); if (hit) { return BufferDescriptorGetBuffer(buf_desc); @@ -243,10 +263,73 @@ void exrto_standby_read_init() if (IS_EXRTO_READ) { make_exrto_file_directory(); } + init_delay_ddl_file(); +} + +bool check_need_drop_buffer(StandbyReadMetaInfo *meta_info, const BufferTag tag) +{ + Assert(meta_info != NULL); + if (!IS_EXRTO_RELFILENODE(tag.rnode)) { + return false; + } + + ExRTOFileType type = exrto_file_type(tag.rnode.spcNode); + if (type == BLOCK_INFO_META) { + return false; + } + + uint32 batch_id = tag.rnode.dbNode >> LOW_WORKERID_BITS; + uint32 worker_id = tag.rnode.dbNode & LOW_WORKERID_MASK; + if (batch_id == meta_info->batch_id && worker_id == meta_info->redo_id) { + uint64 total_block_num = + get_total_block_num(type, tag.rnode.relNode, tag.blockNum); + uint64 recycle_pos = ((type == BASE_PAGE) ? meta_info->base_page_recyle_position + : meta_info->lsn_table_recyle_position); + return (total_block_num < (recycle_pos / BLCKSZ)); + } + + return false; +} + +void buffer_drop_exrto_standby_read_buffers(StandbyReadMetaInfo *meta_info) +{ + bool drop_all = (meta_info == NULL); + if (drop_all) { + ereport(LOG, (errmsg("buffer_drop_exrto_standby_read_buffers: start to drop buffers."))); + } + + int i = 0; + while (i < TOTAL_BUFFER_NUM) { + BufferDesc *buf_desc = GetBufferDescriptor(i); + uint32 buf_state; + bool need_drop = false; + /* + * Some safe unlocked checks can be done to reduce the number of cycle. + */ + if (!IS_EXRTO_RELFILENODE(buf_desc->tag.rnode)) { + i++; + continue; + } + + buf_state = LockBufHdr(buf_desc); + if (drop_all) { + need_drop = IS_EXRTO_RELFILENODE(buf_desc->tag.rnode); + } else { + /* only drop base page and lsn info buffers */ + need_drop = check_need_drop_buffer(meta_info, buf_desc->tag); + } + if (need_drop) { + InvalidateBuffer(buf_desc); /* with buffer head lock released */ + } else { + UnlockBufHdr(buf_desc, buf_state); + } + i++; + } } Datum gs_hot_standby_space_info(PG_FUNCTION_ARGS) { +#ifndef ENABLE_LITE_MODE #define EXRTO_HOT_STANDBY_SPACE_INFO_INFONUM 6 Datum values[EXRTO_HOT_STANDBY_SPACE_INFO_INFONUM]; errno_t rc; @@ -285,8 +368,8 @@ Datum gs_hot_standby_space_info(PG_FUNCTION_ARGS) } for (uint32 i = 0; i < worker_nums; ++i) { - extreme_rto::PageRedoWorker* page_redo_worker = extreme_rto::g_dispatcher->allWorkers[i]; - if (page_redo_worker->role != extreme_rto::REDO_PAGE_WORKER) { + extreme_rto::PageRedoWorker *page_redo_worker = extreme_rto::g_dispatcher->allWorkers[i]; + if (page_redo_worker->role != extreme_rto::REDO_PAGE_WORKER || (page_redo_worker->isUndoSpaceWorker)) { continue; } StandbyReadMetaInfo meta_info = page_redo_worker->standby_read_meta_info; @@ -313,10 +396,10 @@ Datum gs_hot_standby_space_info(PG_FUNCTION_ARGS) basepage_file_size += basepage_file_size_per_thread; } SpinLockRelease(&(g_instance.comm_cxt.predo_cxt.destroy_lock)); - + char block_meta_file_dir[EXRTO_FILE_PATH_LEN]; char block_meta_file_name[EXRTO_FILE_PATH_LEN]; - struct dirent *de = NULL; + struct dirent *de = nullptr; struct stat st; rc = snprintf_s(block_meta_file_dir, EXRTO_FILE_PATH_LEN, EXRTO_FILE_PATH_LEN - 1, "./%s/%s", @@ -347,5 +430,355 @@ Datum gs_hot_standby_space_info(PG_FUNCTION_ARGS) tuple = heap_form_tuple(tupdesc, values, nulls); PG_RETURN_DATUM(HeapTupleGetDatum(tuple)); +#else + FEATURE_ON_LITE_MODE_NOT_SUPPORTED(); + PG_RETURN_TEXT_P(NULL); +#endif } +namespace extreme_rto_standby_read { +typedef struct _DumpLsnInfo { + XLogRecPtr lsn_info_end_lsn; + XLogRecPtr base_page_info_cur_lsn; + std::vector lsn_info_vec; +} DumpLsnInfo; +XLogRecPtr acquire_max_lsn() +{ + if (RecoveryInProgress()) { + ereport(LOG, (errmsg("Can't get local max LSN during recovery in dump."))); + } + XLogRecPtr current_recptr = GetXLogWriteRecPtr(); + return current_recptr; +} + +void exrto_xlog_dump_err_rep(XLogReaderState *xlogreader_state, char *error_msg) +{ + if (xlogreader_state == NULL) { + ereport(LOG, (errmsg("could not read WAL record, xlogreader_state is invalid in dump."))); + return; + } + + if (error_msg != nullptr) { + ereport(LOG, (errmsg("could not read WAL record in dump at %X/%X: %s", + (uint32)(xlogreader_state->ReadRecPtr >> XIDTHIRTYTWO), + (uint32)xlogreader_state->ReadRecPtr, error_msg))); + } else { + ereport(LOG, (errmsg("could not read WAL record in dump at %X/%X", + (uint32)(xlogreader_state->ReadRecPtr >> XIDTHIRTYTWO), + (uint32)xlogreader_state->ReadRecPtr))); + } +} + +void exrto_xlog_dump(char *dump_filename, DumpLsnInfo dump_lsn_info_stru) +{ + XLogRecPtr start_lsn = dump_lsn_info_stru.base_page_info_cur_lsn; + XLogRecPtr end_lsn = dump_lsn_info_stru.lsn_info_end_lsn; + ereport(LOG, (errmsg("start_lsn in dump at %X/%X. end_lsn in dump at %X/%X ", (uint32)(start_lsn >> XIDTHIRTYTWO), + (uint32)start_lsn, (uint32)(end_lsn >> XIDTHIRTYTWO), (uint32)end_lsn))); + /* start reading */ + errno_t rc = EOK; + WalPrivate read_private; + rc = memset_s(&read_private, sizeof(WalPrivate), 0, sizeof(WalPrivate)); + securec_check(rc, "\0", "\0"); + read_private.data_dir = t_thrd.proc_cxt.DataDir; + read_private.tli = 1; + + XLogRecPtr min_lsn = (XLogGetLastRemovedSegno() + 1) * XLogSegSize; + XLogRecPtr max_lsn = acquire_max_lsn(); + if (XLByteLT(start_lsn, min_lsn)) { + start_lsn = min_lsn; + } + if ((max_lsn > start_lsn) && (XLByteLT(max_lsn, end_lsn) || end_lsn == PG_UINT64_MAX)) { + end_lsn = max_lsn; + } + + XLogReaderState *xlogreader_state = XLogReaderAllocate(&SimpleXLogPageRead, &read_private); + if (!xlogreader_state) { + ereport(LOG, (errmsg("memory is temporarily unavailable in dump while allocate xlog reader"))); + } + /* get the first valid xlog record location */ + XLogRecPtr first_record = XLogFindNextRecord(xlogreader_state, start_lsn); + /* if we are recycling or removing log files concurrently, we can't find the next record right after. + * Hence, we need to update the min_lsn */ + if (XLByteEQ(first_record, InvalidXLogRecPtr)) { + ereport(LOG, (errmsg("XLogFindNextRecord in dump: could not find a valid record after %X/%X. Retry.", + (uint32)(start_lsn >> XIDTHIRTYTWO), (uint32)start_lsn))); + bool found = false; + first_record = UpdateNextLSN(start_lsn, end_lsn, xlogreader_state, &found); + if (!found) + ereport(LOG, (errmsg("XLogFindNextRecord in dump: could not find a valid record between %X/%X and %X/%X.", + (uint32)(start_lsn >> XIDTHIRTYTWO), (uint32)start_lsn, + (uint32)(end_lsn >> XIDTHIRTYTWO), (uint32)end_lsn))); + } + XLogRecPtr valid_start_lsn = first_record; + XLogRecPtr valid_end_lsn = valid_start_lsn; + + FILE *output_file = fopen(dump_filename, "a"); + CheckOpenFile(output_file, dump_filename); + char *str_output = (char *)palloc0(MAXOUTPUTLEN * sizeof(char)); + + /* valid first record is not the given one */ + if (!XLByteEQ(first_record, start_lsn) && (start_lsn % XLogSegSize) != 0) { + rc = snprintf_s(str_output + (int)strlen(str_output), MAXOUTPUTLEN, MAXOUTPUTLEN - 1, + "first record is after %X/%X, at %X/%X, skipping over %lu bytes\n", + (uint32)(start_lsn >> XIDTHIRTYTWO), (uint32)start_lsn, + (uint32)(first_record >> XIDTHIRTYTWO), (uint32)first_record, + XLByteDifference(first_record, start_lsn)); + securec_check_ss(rc, "\0", "\0"); + } + CheckWriteFile(output_file, dump_filename, str_output); + pfree_ext(str_output); + + size_t count = 0; + char *error_msg = nullptr; + XLogRecord *record = NULL; + while (xlogreader_state && XLByteLE(xlogreader_state->EndRecPtr, end_lsn)) { + CHECK_FOR_INTERRUPTS(); /* Allow cancel/die interrupts */ + record = XLogReadRecord(xlogreader_state, first_record, &error_msg); + valid_end_lsn = xlogreader_state->EndRecPtr; + if (!record && XLByteLE(valid_end_lsn, end_lsn)) { + /* if we are recycling or removing log files concurrently, and we can't find the next record right + * after. In this case, we try to read from the current oldest xlog file. */ + bool found = false; + XLogRecPtr temp_start_lsn = Max(xlogreader_state->EndRecPtr, start_lsn); + first_record = UpdateNextLSN(temp_start_lsn, end_lsn, xlogreader_state, &found); + if (found) { + ereport(LOG, (errmsg("We cannot read %X/%X. After retried, we jump to read the next available %X/%X. " + "The missing part might be recycled or removed.", + (uint32)(temp_start_lsn >> XIDTHIRTYTWO), (uint32)temp_start_lsn, + (uint32)(first_record >> XIDTHIRTYTWO), (uint32)first_record))); + continue; + } + exrto_xlog_dump_err_rep(xlogreader_state, error_msg); + break; + } else if (!record && !XLByteLT(valid_end_lsn, end_lsn)) { + error_msg = nullptr; + exrto_xlog_dump_err_rep(xlogreader_state, error_msg); + break; + } + + str_output = (char *)palloc(MAXOUTPUTLEN * sizeof(char)); + rc = memset_s(str_output, MAXOUTPUTLEN, 0, MAXOUTPUTLEN); + securec_check(rc, "\0", "\0"); + XLogDumpDisplayRecord(xlogreader_state, str_output); + count++; + + CheckWriteFile(output_file, dump_filename, str_output); + pfree_ext(str_output); + if (count >= dump_lsn_info_stru.lsn_info_vec.size()) { + break; + } + first_record = dump_lsn_info_stru.lsn_info_vec.at(count); + } + XLogReaderFree(xlogreader_state); + str_output = (char *)palloc(MAXOUTPUTLEN * sizeof(char)); + rc = memset_s(str_output, MAXOUTPUTLEN, 0, MAXOUTPUTLEN); + securec_check(rc, "\0", "\0"); + + /* Summary(xx total): valid start_lsn: xxx, valid end_lsn: xxx */ + rc = snprintf_s(str_output + (int)strlen(str_output), MAXOUTPUTLEN, MAXOUTPUTLEN - 1, + "\nSummary (%zu total): valid start_lsn: %X/%X, valid end_lsn: %X/%X\n", count, + (uint32)(valid_start_lsn >> XIDTHIRTYTWO), (uint32)(valid_start_lsn), + (uint32)(valid_end_lsn >> XIDTHIRTYTWO), (uint32)(valid_end_lsn)); + securec_check_ss(rc, "\0", "\0"); + CheckWriteFile(output_file, dump_filename, str_output); + CheckCloseFile(output_file, dump_filename, true); + pfree_ext(str_output); + CloseXlogFile(); +} + +void dump_base_page_info(char *str_output, BasePageInfo base_page_info) +{ + errno_t rc = EOK; + rc = snprintf_s(str_output + (int)strlen(str_output), MAXOUTPUTLEN, MAXOUTPUTLEN - 1, + "Base_page_info: cur_page_lsn=%lu, read_lsn=%lu, spcNode=%u, dbNode=%u, relNode=%u, bucketnode=%d, " + "fork_num=%d, block_num=%u, next_base_page_lsn=%lu, base_page_position=%lu, pre=%lu, next=%lu\n", + base_page_info->cur_page_lsn, t_thrd.proc->exrto_read_lsn, base_page_info->relfilenode.spcNode, + base_page_info->relfilenode.dbNode, base_page_info->relfilenode.relNode, + base_page_info->relfilenode.bucketNode, base_page_info->fork_num, base_page_info->block_num, + base_page_info->next_base_page_lsn, base_page_info->base_page_position, + base_page_info->base_page_list.prev, base_page_info->base_page_list.next); + securec_check_ss(rc, "\0", "\0"); +} + +void dump_lsn_info(char *str_output, const BasePageInfo base_page_info, DumpLsnInfo &dump_lsn_info_stru, + const BufferTag& buf_tag, Buffer buffer) +{ + LsnInfo lsn_info = &base_page_info->lsn_info_node; + LsnInfoPosition next_lsn_info_pos; + bool find_front = false; + dump_lsn_info_stru.base_page_info_cur_lsn = base_page_info->cur_page_lsn; + dump_lsn_info_stru.lsn_info_end_lsn = dump_lsn_info_stru.base_page_info_cur_lsn; + + uint32 batch_id; + uint32 worker_id; + + extreme_rto::RedoItemTag redo_item_tag; + INIT_REDO_ITEM_TAG(redo_item_tag, buf_tag.rnode, buf_tag.forkNum, buf_tag.blockNum); + batch_id = extreme_rto::GetSlotId(buf_tag.rnode, 0, 0, (uint32)extreme_rto::get_batch_redo_num()) + 1; + worker_id = extreme_rto::GetWorkerId(&redo_item_tag, extreme_rto::get_page_redo_worker_num_per_manager()) + 1; + + do { + errno_t rc = EOK; + rc = snprintf_s(str_output + (int)strlen(str_output), MAXOUTPUTLEN, MAXOUTPUTLEN - 1, + "Lsn_info: pre=%lu, next=%lu, flags=%u, type=%u, used=%u\n", + lsn_info->lsn_list.prev, lsn_info->lsn_list.next, lsn_info->flags, lsn_info->type, + lsn_info->used); + securec_check_ss(rc, "\0", "\0"); + for (uint16 i = 0; i < lsn_info->used; ++i) { + rc = snprintf_s(str_output + (int)strlen(str_output), MAXOUTPUTLEN, MAXOUTPUTLEN - 1, + "lsn_num is %lu\n", lsn_info->lsn[i]); + securec_check_ss(rc, "\0", "\0"); + dump_lsn_info_stru.lsn_info_vec.push_back(lsn_info->lsn[i]); + if (!find_front) { + dump_lsn_info_stru.base_page_info_cur_lsn = lsn_info->lsn[i]; + find_front = true; + } + dump_lsn_info_stru.lsn_info_end_lsn = lsn_info->lsn[i]; + } + next_lsn_info_pos = lsn_info->lsn_list.next; + UnlockReleaseBuffer(buffer); + /* reach the end of the list */ + if (next_lsn_info_pos == LSN_INFO_LIST_HEAD) { + break; + } + + Page page = get_lsn_info_page(batch_id, worker_id, next_lsn_info_pos, RBM_NORMAL, &buffer); + if (unlikely(page == NULL || buffer == InvalidBuffer)) { + ereport(LOG, (errmsg(EXRTOFORMAT("get_lsn_info_page failed, batch_id: %u, redo_id: %u, pos: %lu"), batch_id, + worker_id, next_lsn_info_pos))); + break; + } + LockBuffer(buffer, BUFFER_LOCK_SHARE); + + uint32 offset = lsn_info_postion_to_offset(next_lsn_info_pos); + lsn_info = (LsnInfo)(page + offset); + } while (true); +} + +// dump all version of basepage info lsn info +void dump_base_page_info_lsn_info(const BufferTag &buf_tag, LsnInfoPosition head_lsn_base_page_pos, char *str_output, + DumpLsnInfo &dump_lsn_info_stru) +{ + uint32 batch_id; + uint32 worker_id; + BasePageInfo base_page_info = NULL; + Buffer buffer; + + extreme_rto::RedoItemTag redo_item_tag; + INIT_REDO_ITEM_TAG(redo_item_tag, buf_tag.rnode, buf_tag.forkNum, buf_tag.blockNum); + batch_id = extreme_rto::GetSlotId(buf_tag.rnode, 0, 0, (uint32)extreme_rto::get_batch_redo_num()) + 1; + worker_id = extreme_rto::GetWorkerId(&redo_item_tag, extreme_rto::get_page_redo_worker_num_per_manager()) + 1; + + /* find fisrt base page whose lsn less than read lsn form tail to head */ + do { + /* reach the end of the list */ + if (INFO_POSITION_IS_INVALID(head_lsn_base_page_pos)) { + ereport(LOG, (errmsg("can not find base page, block is %u/%u/%u %d %u, batch_id: %u, redo_worker_id: %u", + buf_tag.rnode.spcNode, buf_tag.rnode.dbNode, buf_tag.rnode.relNode, buf_tag.forkNum, + buf_tag.blockNum, batch_id, worker_id))); + break; + } + buffer = InvalidBuffer; + Page page = get_lsn_info_page(batch_id, worker_id, head_lsn_base_page_pos, RBM_NORMAL, &buffer); + if (unlikely(page == NULL || buffer == InvalidBuffer)) { + ereport(LOG, (errmsg(EXRTOFORMAT("get_lsn_info_page failed, batch_id: %u, redo_id: %u, pos: %lu"), batch_id, + worker_id, head_lsn_base_page_pos))); + break; + } + LockBuffer(buffer, BUFFER_LOCK_SHARE); + + uint32 offset = lsn_info_postion_to_offset(head_lsn_base_page_pos); + base_page_info = (BasePageInfo)(page + offset); + + if (is_base_page_type(base_page_info->lsn_info_node.type) == false) { + UnlockReleaseBuffer(buffer); + break; + } + + Buffer base_page_datum_buffer = + buffer_read_base_page(batch_id, worker_id, base_page_info->base_page_position, RBM_NORMAL); + LockBuffer(base_page_datum_buffer, BUFFER_LOCK_SHARE); + + UnlockReleaseBuffer(base_page_datum_buffer); + + dump_base_page_info(str_output, base_page_info); // print info + dump_lsn_info(str_output, base_page_info, dump_lsn_info_stru, buf_tag, buffer); + head_lsn_base_page_pos = base_page_info->base_page_list.prev; + } while (true); +} + +void dump_current_lsn(char *str_output) +{ + errno_t rc = snprintf_s(str_output + (int)strlen(str_output), MAXOUTPUTLEN, MAXOUTPUTLEN - 1, + "current redo position %lu, cache lsn: %lu\n", + g_instance.comm_cxt.predo_cxt.redoPf.last_replayed_end_ptr, t_thrd.proc->exrto_read_lsn); + securec_check_ss(rc, "\0", "\0"); +} + +void dump_one_block_info(char *str_output, BlockMetaInfo* block_meta_info) +{ + errno_t rc = snprintf_s(str_output + (int)strlen(str_output), MAXOUTPUTLEN, MAXOUTPUTLEN - 1, + "Block info meta info: timeline=%u, record_num=%u, min_lsn=%lu, max_lsn=%lu, " + "lsn_info_list prev=%lu, next=%lu, base_page_info_list prev=%lu, next=%lu\n", + block_meta_info->timeline, block_meta_info->record_num, block_meta_info->min_lsn, + block_meta_info->max_lsn, block_meta_info->lsn_info_list.prev, + block_meta_info->lsn_info_list.next, block_meta_info->base_page_info_list.prev, + block_meta_info->base_page_info_list.next); + securec_check_ss(rc, "\0", "\0"); +} + +void dump_error_all_info(const RelFileNode &rnode, ForkNumber forknum, BlockNumber blocknum) +{ + if (!IS_EXRTO_STANDBY_READ) { + return; + } + buffer_in_progress_pop(); + BufferTag buf_tag; + INIT_BUFFERTAG(buf_tag, rnode, forknum, blocknum); + + Buffer buf; + BlockMetaInfo *block_meta_info = get_block_meta_info_by_relfilenode(buf_tag, NULL, RBM_NORMAL, &buf, true); + if (block_meta_info == NULL) { + ereport(LOG, + (errmsg("can not find block meta info by given buftag. rnode is %u/%u/%u %d %u", buf_tag.rnode.spcNode, + buf_tag.rnode.dbNode, buf_tag.rnode.relNode, buf_tag.forkNum, buf_tag.blockNum))); + buffer_in_progress_push(); + return; // it's more likely we cannot get block info than cannot alloc file, so put this first for performance. + } + + char *str_output = (char *)palloc0(MAXOUTPUTLEN * sizeof(char)); + char *dump_filename = (char *)palloc0(MAXFILENAME * sizeof(char)); + errno_t rc = snprintf_s(dump_filename + (int)strlen(dump_filename), MAXFILENAME, MAXFILENAME - 1, + "%s/%u_%u_%u_%d_%d.lsnblockinfo_dump", t_thrd.proc_cxt.DataDir, rnode.spcNode, rnode.dbNode, rnode.relNode, + forknum, blocknum); + securec_check_ss(rc, "\0", "\0"); + FILE *dump_file = AllocateFile(dump_filename, PG_BINARY_W); + if (dump_file == NULL) { + ereport(LOG, (errmsg("can not alloc file. rnode is %u/%u/%u %d %u", buf_tag.rnode.spcNode, + buf_tag.rnode.dbNode, buf_tag.rnode.relNode, buf_tag.forkNum, buf_tag.blockNum))); + pfree_ext(str_output); + pfree_ext(dump_filename); + buffer_in_progress_push(); + return; + } + + dump_current_lsn(str_output); + dump_one_block_info(str_output, block_meta_info); + + DumpLsnInfo dump_lsn_info_stru; + dump_lsn_info_stru.lsn_info_end_lsn = PG_UINT64_MAX; + dump_base_page_info_lsn_info(buf_tag, block_meta_info->base_page_info_list.prev, str_output, dump_lsn_info_stru); + UnlockReleaseBuffer(buf); // buf was automatically locked by getting block meta info, so we need release + + uint result = fwrite(str_output, 1, strlen(str_output), dump_file); + if (result != strlen(str_output)) { + ereport(ERROR, (errcode(ERRCODE_FILE_WRITE_FAILED), errmsg("Cannot write into file %s!", dump_filename))); + } + pfree_ext(str_output); + (void)FreeFile(dump_file); + exrto_xlog_dump(dump_filename, dump_lsn_info_stru); + pfree_ext(dump_filename); + buffer_in_progress_push(); +} +} diff --git a/src/gausskernel/storage/access/spgist/spgxlog.cpp b/src/gausskernel/storage/access/spgist/spgxlog.cpp index 8f3aa009c..b438d14b0 100755 --- a/src/gausskernel/storage/access/spgist/spgxlog.cpp +++ b/src/gausskernel/storage/access/spgist/spgxlog.cpp @@ -497,7 +497,7 @@ static void spgRedoVacuumRedirect(XLogReaderState *record) * If any redirection tuples are being removed, make sure there are no * live Hot Standby transactions that might need to see them. */ - if (InHotStandby && g_supportHotStandby) { + if (InHotStandby && g_supportHotStandby && !IS_EXRTO_READ) { spgxlogVacuumRedirect *xldata = (spgxlogVacuumRedirect *)ptr; if (TransactionIdIsValid(xldata->newestRedirectXid)) { RelFileNode node; diff --git a/src/gausskernel/storage/access/transam/clog.cpp b/src/gausskernel/storage/access/transam/clog.cpp index a0a050d93..0decff72f 100644 --- a/src/gausskernel/storage/access/transam/clog.cpp +++ b/src/gausskernel/storage/access/transam/clog.cpp @@ -1130,6 +1130,7 @@ void clog_redo(XLogReaderState *record) Assert(!ClogCtl(pageno)->shared->page_dirty[slotno]); LWLockRelease(ClogCtl(pageno)->shared->control_lock); + g_instance.comm_cxt.predo_cxt.max_clog_pageno = pageno; } else if (info == CLOG_TRUNCATE) { int64 pageno; diff --git a/src/gausskernel/storage/access/transam/extreme_rto/dispatcher.cpp b/src/gausskernel/storage/access/transam/extreme_rto/dispatcher.cpp index 61e853c84..d24d2fdbd 100755 --- a/src/gausskernel/storage/access/transam/extreme_rto/dispatcher.cpp +++ b/src/gausskernel/storage/access/transam/extreme_rto/dispatcher.cpp @@ -118,6 +118,7 @@ static void SSDestroyRecoveryWorkers(); static void DispatchRecordWithPages(XLogReaderState *, List *); static void DispatchRecordWithoutPage(XLogReaderState *, List *); static void DispatchTxnRecord(XLogReaderState *, List *); +void dispatch_record_to_all_thread(XLogReaderState *record, List *expected_tlis); static void StartPageRedoWorkers(uint32); static void StopRecoveryWorkers(int, Datum); static bool StandbyWillChangeStandbyState(const XLogReaderState *); @@ -639,6 +640,8 @@ void SendSingalToPageWorker(int signal) for (uint32 i = 0; i < g_instance.comm_cxt.predo_cxt.totalNum; ++i) { uint32 state = pg_atomic_read_u32(&(g_instance.comm_cxt.predo_cxt.pageRedoThreadStatusList[i].threadState)); if (state == PAGE_REDO_WORKER_READY) { + ereport(LOG, (errmsg("Dispatch start to kill(pid %lu, signal %d)", + g_instance.comm_cxt.predo_cxt.pageRedoThreadStatusList[i].threadId, signal))); int err = gs_signal_send(g_instance.comm_cxt.predo_cxt.pageRedoThreadStatusList[i].threadId, signal); if (0 != err) { ereport(WARNING, (errmsg("Dispatch kill(pid %lu, signal %d) failed: \"%s\",", @@ -675,7 +678,7 @@ static void StopRecoveryWorkers(int code, Datum arg) if ((count & OUTPUT_WAIT_COUNT) == OUTPUT_WAIT_COUNT) { ereport(WARNING, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), errmsg("StopRecoveryWorkers wait page work exit"))); - if ((count & PRINT_ALL_WAIT_COUNT) == PRINT_ALL_WAIT_COUNT) { + if ((count & STOP_WORKERS_WAIT_COUNT) == STOP_WORKERS_WAIT_COUNT) { DumpDispatcher(); ereport(PANIC, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), errmsg("StopRecoveryWorkers wait too long!!!"))); @@ -700,8 +703,8 @@ static void StopRecoveryWorkers(int code, Datum arg) /* Run from the dispatcher thread. */ static void DestroyRecoveryWorkers() { + SpinLockAcquire(&(g_instance.comm_cxt.predo_cxt.destroy_lock)); if (g_dispatcher != NULL) { - SpinLockAcquire(&(g_instance.comm_cxt.predo_cxt.destroy_lock)); for (uint32 i = 0; i < g_dispatcher->pageLineNum; i++) { DestroyPageRedoWorker(g_dispatcher->pageLines[i].batchThd); DestroyPageRedoWorker(g_dispatcher->pageLines[i].managerThd); @@ -734,8 +737,8 @@ static void DestroyRecoveryWorkers() g_instance.comm_cxt.predo_cxt.parallelRedoCtx = NULL; } g_dispatcher = NULL; - SpinLockRelease(&(g_instance.comm_cxt.predo_cxt.destroy_lock)); } + SpinLockRelease(&(g_instance.comm_cxt.predo_cxt.destroy_lock)); } static void SSDestroyRecoveryWorkers() @@ -895,7 +898,8 @@ static void DispatchSyncTxnRecord(XLogReaderState *record, List *expectedTLIs) RedoItem *item = GetRedoItemPtr(record); ReferenceRedoItem(item); - if ((g_dispatcher->chosedPLCnt != 1) && (XLogRecGetRmid(&item->record) != RM_XACT_ID)) { + if ((g_dispatcher->chosedPLCnt != 1) && (XLogRecGetRmid(&item->record) != RM_XACT_ID) && + !(IsSmgrTruncate(&item->record))) { ereport(WARNING, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), errmsg("[REDO_LOG_TRACE]DispatchSyncTxnRecord maybe some error:rmgrID:%u, info:%u, workerCount:%u", @@ -1025,6 +1029,8 @@ static bool DispatchXLogRecord(XLogReaderState *record, List *expectedTLIs, Time } else if ((info == XLOG_FPI) || (info == XLOG_FPI_FOR_HINT)) { DispatchRecordWithPages(record, expectedTLIs); + } else if (info == XLOG_BACKUP_END) { + dispatch_record_to_all_thread(record, expectedTLIs); } else { /* process in trxn thread and need to sync to other pagerredo thread */ DispatchTxnRecord(record, expectedTLIs); @@ -1033,6 +1039,17 @@ static bool DispatchXLogRecord(XLogReaderState *record, List *expectedTLIs, Time return isNeedFullSync; } +void dispatch_record_to_all_thread(XLogReaderState *record, List *expected_tlis) +{ + RedoItem *item = GetRedoItemPtr(record); + ReferenceRedoItem(item); + for (uint32 i = 0; i < g_dispatcher->pageLineNum; i++) { + ReferenceRedoItem(item); + AddPageRedoItem(g_dispatcher->pageLines[i].batchThd, item); + } + AddTxnRedoItem(g_dispatcher->trxnLine.managerThd, item); +} + /* Run from the dispatcher thread. */ static bool DispatchRelMapRecord(XLogReaderState *record, List *expectedTLIs, TimestampTz recordXTime) { @@ -1044,7 +1061,7 @@ static bool DispatchRelMapRecord(XLogReaderState *record, List *expectedTLIs, Ti /* Run from the dispatcher thread. */ static bool DispatchXactRecord(XLogReaderState *record, List *expectedTLIs, TimestampTz recordXTime) { - if (XactWillRemoveRelFiles(record)) { + if (xact_has_invalid_msg_or_delete_file(record)) { bool hasSegpageRelFile = XactHasSegpageRelFiles(record); uint32 doneFlag = 0; @@ -1207,15 +1224,9 @@ static bool DispatchSmgrRecord(XLogReaderState *record, List *expectedTLIs, Time RelFileNodeCopy(rnode, xlrec->rnode, XLogRecGetBucketId(record)); rnode.opt = GetCreateXlogFileNodeOpt(record); DispatchToOnePageWorker(record, rnode, expectedTLIs); - } else if (IsSmgrTruncate(record)) { - xl_smgr_truncate *xlrec = (xl_smgr_truncate *)XLogRecGetData(record); - RelFileNode rnode; - RelFileNodeCopy(rnode, xlrec->rnode, XLogRecGetBucketId(record)); - rnode.opt = GetTruncateXlogFileNodeOpt(record); - uint32 id = GetSlotId(rnode, 0, 0, GetBatchCount()); - AddSlotToPLSet(id); - - DispatchToSpecPageWorker(record, expectedTLIs); + } else if (info == XLOG_SMGR_TRUNCATE) { + record->isFullSync = true; + dispatch_record_to_all_thread(record, expectedTLIs); } return isNeedFullSync; diff --git a/src/gausskernel/storage/access/transam/extreme_rto/exrto_recycle.cpp b/src/gausskernel/storage/access/transam/extreme_rto/exrto_recycle.cpp index 30f94c2a3..18c47d565 100644 --- a/src/gausskernel/storage/access/transam/extreme_rto/exrto_recycle.cpp +++ b/src/gausskernel/storage/access/transam/extreme_rto/exrto_recycle.cpp @@ -26,6 +26,8 @@ #include "access/extreme_rto/page_redo.h" #include "access/extreme_rto/dispatcher.h" #include "access/extreme_rto/standby_read/lsn_info_meta.h" +#include "access/extreme_rto/standby_read.h" +#include "access/extreme_rto/standby_read/standby_read_delay_ddl.h" #include "access/multi_redo_api.h" #include "storage/ipc.h" #include "storage/smgr/smgr.h" @@ -103,12 +105,17 @@ bool check_if_need_force_recycle() PageRedoWorker** workers = g_dispatcher->allWorkers; int64 total_base_page_size = 0; int64 total_lsn_info_size = 0; - double ratio = g_instance.attr.attr_storage.standby_force_recyle_ratio; + double ratio = g_instance.attr.attr_storage.standby_force_recycle_ratio; + + // if standby_force_recyle_ratio is 0, the system does not recyle file. + if (ratio == 0) { + return false; + } for (uint32 i = 0; i < worker_nums; ++i) { PageRedoWorker* page_redo_worker = workers[i]; StandbyReadMetaInfo meta_info = page_redo_worker->standby_read_meta_info; - if (page_redo_worker->role != REDO_PAGE_WORKER) { + if (page_redo_worker->role != REDO_PAGE_WORKER || (page_redo_worker->isUndoSpaceWorker)) { continue; } total_base_page_size += (meta_info.base_page_next_position - meta_info.base_page_recyle_position); @@ -130,7 +137,7 @@ void do_standby_read_recyle(XLogRecPtr recycle_lsn) XLogRecPtr min_recycle_lsn = InvalidXLogRecPtr; for (uint32 i = 0; i < worker_nums; ++i) { PageRedoWorker* page_redo_worker = workers[i]; - if (page_redo_worker->role != REDO_PAGE_WORKER) { + if (page_redo_worker->role != REDO_PAGE_WORKER || (page_redo_worker->isUndoSpaceWorker)) { continue; } extreme_rto_standby_read::standby_read_recyle_per_workers(&page_redo_worker->standby_read_meta_info, recycle_lsn); @@ -145,6 +152,7 @@ void do_standby_read_recyle(XLogRecPtr recycle_lsn) (errmsg(EXRTOFORMAT("[exrto_recycle] update global recycle lsn: %08X/%08X"), (uint32)(min_recycle_lsn >> UINT64_HALF), (uint32)min_recycle_lsn))); } + delete_by_lsn(recycle_lsn); } void exrto_recycle_interrupt() @@ -197,6 +205,7 @@ void exrto_recycle_main() ereport(LOG, (errmsg("exrto recycle: standby_read_old dir not exist"))); } + do_all_old_delay_ddl(); if (!IS_EXRTO_READ || !RecoveryInProgress()) { ereport(LOG, (errmsg("exrto recycle is available only when exrto standby read is supported"))); diff --git a/src/gausskernel/storage/access/transam/extreme_rto/page_redo.cpp b/src/gausskernel/storage/access/transam/extreme_rto/page_redo.cpp index 24f8a54f0..7d5a19da6 100755 --- a/src/gausskernel/storage/access/transam/extreme_rto/page_redo.cpp +++ b/src/gausskernel/storage/access/transam/extreme_rto/page_redo.cpp @@ -23,7 +23,7 @@ * an idle worker to "steal" work from a busy worker. * * IDENTIFICATION - * src/gausskernel/storage/access/transam/parallel_recovery/page_redo.cpp + * src/gausskernel/storage/access/transam/extreme_rto/page_redo.cpp * * ------------------------------------------------------------------------- */ @@ -41,6 +41,7 @@ #include "access/xlogutils.h" #include "access/xlogproc.h" #include "access/nbtree.h" +#include "access/multi_redo_api.h" #include "catalog/storage_xlog.h" #include "gssignal/gs_signal.h" #include "libpq/pqsignal.h" @@ -74,6 +75,7 @@ #include "replication/datareceiver.h" #include "pgxc/barrier.h" #include "storage/file/fio_device.h" +#include "utils/timestamp.h" #ifdef ENABLE_MOT #include "storage/mot/mot_fdw.h" #endif @@ -98,6 +100,7 @@ namespace extreme_rto { static const int MAX_PARSE_BUFF_NUM = PAGE_WORK_QUEUE_SIZE * 10 * 3; static const int MAX_LOCAL_BUFF_NUM = PAGE_WORK_QUEUE_SIZE * 10 * 3; +static const int MAX_CLEAR_SMGR_NUM = 100000; static const char *const PROCESS_TYPE_CMD_ARG = "--forkpageredo"; static char g_AUXILIARY_TYPE_CMD_ARG[16] = {0}; @@ -350,7 +353,7 @@ void HandlePageRedoPageRepair(RepairBlockKey key, XLogPhyBlock pblk) RecordBadBlockAndPushToRemote(g_redoWorker->curRedoBlockState, CRC_CHECK_FAIL, InvalidXLogRecPtr, pblk); } -void HandlePageRedoInterrupts() +void HandlePageRedoInterruptsImpl(uint64 clearRedoFdCountInc = 1) { if (t_thrd.page_redo_cxt.got_SIGHUP) { t_thrd.page_redo_cxt.got_SIGHUP = false; @@ -371,6 +374,23 @@ void HandlePageRedoInterrupts() proc_exit(1); } + + static uint64 clearRedoFdCount = 0; + const uint64 clearRedoFdCountMask = 0x7FFFFFF; + clearRedoFdCount += clearRedoFdCountInc; + if (clearRedoFdCount > clearRedoFdCountMask && GetSMgrRelationHash() != NULL && + (g_redoWorker->role == REDO_PAGE_WORKER || g_redoWorker->role == REDO_PAGE_MNG)) { + clearRedoFdCount = 0; + long hash_num = hash_get_num_entries(GetSMgrRelationHash()); + if (hash_num >= MAX_CLEAR_SMGR_NUM) { + smgrcloseall(); + } + } +} + +void HandlePageRedoInterrupts() +{ + HandlePageRedoInterruptsImpl(); } void ReferenceRedoItem(void *item) @@ -796,7 +816,16 @@ void RedoPageManagerDoSmgrAction(XLogRecParseState *recordblockstate) (void)MemoryContextSwitchTo(oldCtx); recordblockstate->nextrecord = NULL; + bool need_wait = recordblockstate->isFullSync; + if (need_wait) { + pg_atomic_write_u32(&g_redoWorker->fullSyncFlag, 1); + } XLogBlockParseStateRelease(recordblockstate); + uint32 val = pg_atomic_read_u32(&g_redoWorker->fullSyncFlag); + while (val != 0) { + RedoInterruptCallBack(); + val = pg_atomic_read_u32(&g_redoWorker->fullSyncFlag); + } } void RedoPageManagerDoDataTypeAction(XLogRecParseState *parsestate) @@ -872,6 +901,19 @@ void PageManagerProcCheckPoint(XLogRecParseState *parseState) } } +void page_manager_proc_common_type(XLogRecParseState *parse_state) +{ + if (IsCheckPoint(parse_state)) { + PageManagerProcCheckPoint(parse_state); + } else if (is_backup_end(parse_state)) { + RedoPageManagerDistributeToAllOneBlock(parse_state); + XLogBlockParseStateRelease(parse_state); + } else { + Assert(0); + XLogBlockParseStateRelease(parse_state); + } +} + void PageManagerProcCreateTableSpace(XLogRecParseState *parseState) { bool needWait = parseState->isFullSync; @@ -970,7 +1012,7 @@ void PageManagerRedoParseState(XLogRecParseState *preState) CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_7]); break; case BLOCK_DATA_XLOG_COMMON_TYPE: - PageManagerProcCheckPoint(preState); + page_manager_proc_common_type(preState); break; case BLOCK_DATA_NEWCU_TYPE: PageManagerDistributeBcmBlock(preState); @@ -987,6 +1029,10 @@ void PageManagerRedoParseState(XLogRecParseState *preState) RedoPageManagerDistributeToAllOneBlock(preState); WaitNextBarrier(preState); break; + case BLOCK_DATA_XACTDATA_TYPE: + RedoPageManagerDistributeToAllOneBlock(preState); + XLogBlockParseStateRelease(preState); + break; default: XLogBlockParseStateRelease(preState); break; @@ -998,10 +1044,10 @@ bool PageManagerRedoDistributeItems(XLogRecParseState *record_block_state) if (record_block_state == (void *)&g_redoEndMark) { return true; } else if (record_block_state == (void *)&g_GlobalLsnForwarder) { - PageManagerProcLsnForwarder((RedoItem *) record_block_state); + PageManagerProcLsnForwarder((RedoItem *)record_block_state); return false; } else if (record_block_state == (void *)&g_cleanupMark) { - PageManagerProcCleanupMark((RedoItem *) record_block_state); + PageManagerProcCleanupMark((RedoItem *)record_block_state); return false; } else if (record_block_state == (void *)&g_cleanInvalidPageMark) { forget_range_invalid_pages((void *)record_block_state); @@ -1030,28 +1076,34 @@ bool PageManagerRedoDistributeItems(XLogRecParseState *record_block_state) void RedoPageManagerMain() { - XLogRecParseState *record_block_state; - bool is_end; + XLogRecParseState *record_block_state = NULL; (void)RegisterRedoInterruptCallBack(HandlePageRedoInterrupts); XLogParseBufferInitFunc(&(g_redoWorker->parseManager), MAX_PARSE_BUFF_NUM, &recordRefOperate, RedoInterruptCallBack); GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_1]); - while (true) { - if (!SPSCBlockingQueueIsEmpty(g_redoWorker->queue)) { - record_block_state = (XLogRecParseState *)SPSCBlockingQueueTake(g_redoWorker->queue); - CountAndGetRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_1], g_redoWorker->timeCostList[TIME_COST_STEP_2]); - is_end = PageManagerRedoDistributeItems(record_block_state); - CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_2]); - if (is_end) - break; - } + while ((record_block_state = (XLogRecParseState *)SPSCBlockingQueueTop(g_redoWorker->queue)) != + (XLogRecParseState *)&g_redoEndMark) { + ErrorContextCallback err_context; + err_context.callback = rm_redo_error_callback; + err_context.arg = (void *)record_block_state->refrecord; + err_context.previous = t_thrd.log_cxt.error_context_stack; + t_thrd.log_cxt.error_context_stack = &err_context; + + CountAndGetRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_1], + g_redoWorker->timeCostList[TIME_COST_STEP_2]); + PageManagerRedoDistributeItems(record_block_state); + SPSCBlockingQueuePop(g_redoWorker->queue); + CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_2]); RedoInterruptCallBack(); ADD_ABNORMAL_POSITION(5); GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_1]); + + t_thrd.log_cxt.error_context_stack = err_context.previous; } + SPSCBlockingQueuePop(g_redoWorker->queue); RedoThrdWaitForExit(g_redoWorker); XLogParseBufferDestoryFunc(&(g_redoWorker->parseManager)); } @@ -1095,8 +1147,9 @@ bool TrxnManagerDistributeItemsBeforeEnd(RedoItem *item) } else { GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_4]); if (IsCheckPoint(&item->record) || IsTableSpaceDrop(&item->record) || IsTableSpaceCreate(&item->record) || - (IsXactXlog(&item->record) && XactWillRemoveRelFiles(&item->record)) || IsBarrierRelated(&item->record) || - IsDataBaseDrop(&item->record) || IsDataBaseCreate(&item->record)) { + (IsXactXlog(&item->record) && xact_has_invalid_msg_or_delete_file(&item->record)) || + IsBarrierRelated(&item->record) || IsDataBaseDrop(&item->record) || IsDataBaseCreate(&item->record) || + IsSmgrTruncate(&item->record)) { uint32 relCount; do { RedoInterruptCallBack(); @@ -1262,6 +1315,10 @@ void TrxnWorkMain() forget_range_invalid_pages((void *)item); SPSCBlockingQueuePop(g_redoWorker->queue); } else { + if (IsSmgrTruncate(&item->record)) { + // need generate a new snapshot before truncate, and lsn is larger than the actual value + exrto_generate_snapshot(item->record.EndRecPtr); + } t_thrd.xlog_cxt.needImmediateCkp = item->needImmediateCheckpoint; bool fullSync = item->record.isFullSync; GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_3]); @@ -1278,7 +1335,7 @@ void TrxnWorkMain() } if (IsCheckPoint(&item->record) || (IsXactXlog(&item->record) && - XactWillRemoveRelFiles(&item->record)) || IsBarrierRelated(&item->record) || + xact_has_invalid_msg_or_delete_file(&item->record)) || IsBarrierRelated(&item->record) || IsDataBaseDrop(&item->record)) { exrto_generate_snapshot(g_redoWorker->lastReplayedEndRecPtr); } @@ -1394,6 +1451,13 @@ void RedoPageWorkerRedoBcmBlock(XLogRecParseState *procState) } } +void redo_page_worker_proc_common_record(XLogRecParseState *stat) +{ + if (IsCheckPoint(stat)) { + RedoPageWorkerCheckPoint(stat); + } +} + void RedoPageWorkerMain() { (void)RegisterRedoInterruptCallBack(HandlePageRedoInterrupts); @@ -1437,6 +1501,12 @@ void RedoPageWorkerMain() g_redoWorker->curRedoBlockState = (XLogBlockDataParse*)(&redoblockstate->blockparse.extra_rec); procState = (XLogRecParseState *)procState->nextrecord; + ErrorContextCallback err_context; + err_context.callback = rm_redo_error_callback; + err_context.arg = (void *)redoblockstate->refrecord; + err_context.previous = t_thrd.log_cxt.error_context_stack; + t_thrd.log_cxt.error_context_stack = &err_context; + switch (XLogBlockHeadGetValidInfo(&redoblockstate->blockparse.blockhead)) { case BLOCK_DATA_MAIN_DATA_TYPE: case BLOCK_DATA_UNDO_TYPE: @@ -1449,7 +1519,7 @@ void RedoPageWorkerMain() break; case BLOCK_DATA_XLOG_COMMON_TYPE: GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); - RedoPageWorkerCheckPoint(redoblockstate); + redo_page_worker_proc_common_record(redoblockstate); SetCompletedReadEndPtr(g_redoWorker, redoblockstate->blockparse.blockhead.start_ptr, redoblockstate->blockparse.blockhead.end_ptr); CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); @@ -1492,9 +1562,16 @@ void RedoPageWorkerMain() redoblockstate->blockparse.blockhead.end_ptr); CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); break; + case BLOCK_DATA_XACTDATA_TYPE: + GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); + SetCompletedReadEndPtr(g_redoWorker, redoblockstate->blockparse.blockhead.start_ptr, + redoblockstate->blockparse.blockhead.end_ptr); + CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); + break; default: break; } + t_thrd.log_cxt.error_context_stack = err_context.previous; } (void)MemoryContextSwitchTo(oldCtx); GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_7]); @@ -1570,14 +1647,11 @@ void PushToWorkerLsn(bool force) { const uint32 max_record_count = PAGE_WORK_QUEUE_SIZE; static uint32 cur_recor_count = 0; - - cur_recor_count++; - if (!IsExtremeRtoRunning()) { return; } - - if (force) { + cur_recor_count++; + if (unlikely(force)) { uint32 refCount; do { refCount = pg_atomic_read_u32(&g_GlobalLsnForwarder.record.refcount); @@ -1586,17 +1660,36 @@ void PushToWorkerLsn(bool force) cur_recor_count = 0; SendLsnFowarder(); } else { - uint32 refCount = pg_atomic_read_u32(&g_GlobalLsnForwarder.record.refcount); - - if (refCount != 0 || cur_recor_count < max_record_count) { + if (cur_recor_count < max_record_count) { return; } + if (pg_atomic_read_u32(&g_GlobalLsnForwarder.record.refcount) != 0) { + return; + } SendLsnFowarder(); cur_recor_count = 0; } } +inline bool send_lsn_forwarder_for_check_to_hot_standby(XLogRecPtr lsn) +{ + if (t_thrd.xlog_cxt.reachedConsistency) { + // means has send lsn forwarder for consistenstcy check + return false; + } + if (XLogRecPtrIsInvalid(t_thrd.xlog_cxt.minRecoveryPoint)) { + return false; + } + + if (XLByteLT(lsn, t_thrd.xlog_cxt.minRecoveryPoint)) { + return false; + } + + t_thrd.xlog_cxt.reachedConsistency = true; + return true; +} + void ResetRtoXlogReadBuf(XLogRecPtr targetPagePtr) { uint32 startreadworker = pg_atomic_read_u32(&(g_recordbuffer->readWorkerState)); @@ -1975,14 +2068,14 @@ void XLogReadPageWorkerMain() g_redoWorker->lastReplayedReadRecPtr = xlogreader->ReadRecPtr; g_redoWorker->lastReplayedEndRecPtr = xlogreader->EndRecPtr; - + PushToWorkerLsn(send_lsn_forwarder_for_check_to_hot_standby(g_redoWorker->lastReplayedEndRecPtr)); + if (FORCE_FINISH_ENABLED) { CheckAndDoForceFinish(xlogreader); } CountAndGetRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_5], g_redoWorker->timeCostList[TIME_COST_STEP_1]); record = XLogParallelReadNextRecord(xlogreader); CountAndGetRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_1], g_redoWorker->timeCostList[TIME_COST_STEP_2]); - PushToWorkerLsn(false); CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_2]); RedoInterruptCallBack(); ADD_ABNORMAL_POSITION(8); @@ -2739,8 +2832,9 @@ bool XactHasSegpageRelFiles(XLogReaderState *record) if (XLogRecGetRmid(record) != RM_XACT_ID) { return false; } - bool compress; - XactGetRelFiles(record, &xnodes, &nrels, &compress); + + bool compress = (bool)(XLogRecGetInfo(record) & XLR_REL_COMPRESS); + XactGetRelFiles(record, &xnodes, &nrels); for (int32 idx = 0; idx < nrels; idx++) { ColFileNode colFileNode; @@ -3006,70 +3100,73 @@ void SeqCheckRemoteReadAndRepairPage() } } +inline void invalid_msg_leak_warning(XLogRecPtr trxn_lsn) +{ + if (t_thrd.page_redo_cxt.invalid_msg.valid) { + ereport(WARNING, (errmsg(EXRTOFORMAT("[exrto_generate_snapshot] not send invalid msg: %08X/%08X"), + (uint32)(trxn_lsn >> UINT64_HALF), (uint32)trxn_lsn))); + } +} + void exrto_generate_snapshot(XLogRecPtr trxn_lsn) { if (!g_instance.attr.attr_storage.EnableHotStandby) { return; } - ExrtoSnapshot exrto_snapshot = &g_dispatcher->exrto_snapshot; + ExrtoSnapshot exrto_snapshot = g_instance.comm_cxt.predo_cxt.exrto_snapshot; /* * do not generate the same snapshot repeatedly. */ if (XLByteLE(trxn_lsn, exrto_snapshot->read_lsn)) { + invalid_msg_leak_warning(trxn_lsn); return; } + + TransactionId xmin; + TransactionId xmax; + CommitSeqNo snapshot_csn; - if (XLogRecPtrIsInvalid(t_thrd.xlog_cxt.minRecoveryPoint)) { - return; - } - if (XLByteLT(trxn_lsn, exrto_snapshot->read_lsn)) { - return; - } - - SnapshotData snapshot; - - (void)GetSnapshotData(&snapshot, false); - - Assert(snapshot.takenDuringRecovery); + exrto_get_snapshot_data(xmin, xmax, snapshot_csn); (void)LWLockAcquire(ExrtoSnapshotLock, LW_EXCLUSIVE); - exrto_snapshot->snapshot_csn = snapshot.snapshotcsn; - exrto_snapshot->xmin = snapshot.xmin; - exrto_snapshot->xmax = snapshot.xmax; + exrto_snapshot->snapshot_csn = snapshot_csn; + exrto_snapshot->xmin = xmin; + exrto_snapshot->xmax = xmax; exrto_snapshot->read_lsn = trxn_lsn; - exrto_snapshot->gen_snap_time = GetCurrentTimestamp(); + send_delay_invalid_message(); LWLockRelease(ExrtoSnapshotLock); } void exrto_read_snapshot(Snapshot snapshot) { - if (t_thrd.role != WORKER && t_thrd.role != THREADPOOL_WORKER) { + if ((!is_exrto_standby_read_worker()) || u_sess->proc_cxt.clientIsCMAgent || dummyStandbyMode) { return; } - - if (g_dispatcher == NULL) { - ereport(ERROR, - (errmsg("g_dispatcher is not init")));; - } - - ExrtoSnapshot exrto_snapshot = &g_dispatcher->exrto_snapshot; + + ExrtoSnapshot exrto_snapshot = g_instance.comm_cxt.predo_cxt.exrto_snapshot; (void)LWLockAcquire(ExrtoSnapshotLock, LW_SHARED); if (XLByteEQ(exrto_snapshot->read_lsn, 0)) { LWLockRelease(ExrtoSnapshotLock); - ereport(ERROR, - (errmsg("could not get a valid snapshot with extreme rto"))); + ereport(ERROR, (errmsg("could not get a valid snapshot with extreme rto"))); } snapshot->snapshotcsn = exrto_snapshot->snapshot_csn; snapshot->xmin = exrto_snapshot->xmin; snapshot->xmax = exrto_snapshot->xmax; - - t_thrd.pgxact->xmin = exrto_snapshot->xmin; - t_thrd.proc->exrto_read_lsn = exrto_snapshot->read_lsn; - t_thrd.proc->exrto_gen_snap_time = exrto_snapshot->gen_snap_time; - u_sess->utils_cxt.TransactionXmin = exrto_snapshot->xmin; - u_sess->utils_cxt.exrto_read_lsn = exrto_snapshot->read_lsn; - + snapshot->read_lsn = exrto_snapshot->read_lsn; LWLockRelease(ExrtoSnapshotLock); + if (!TransactionIdIsValid(t_thrd.pgxact->xmin) || TransactionIdPrecedes(snapshot->xmin, t_thrd.pgxact->xmin)) { + t_thrd.pgxact->xmin = snapshot->xmin; + u_sess->utils_cxt.TransactionXmin = snapshot->xmin; + } + t_thrd.proc->exrto_read_lsn = exrto_snapshot->read_lsn; + if (t_thrd.proc->exrto_min == 0 || + XLByteLT(t_thrd.proc->exrto_min, t_thrd.proc->exrto_read_lsn)) { + t_thrd.proc->exrto_min = t_thrd.proc->exrto_read_lsn; + } + + if (t_thrd.proc->exrto_gen_snap_time == 0) { + t_thrd.proc->exrto_gen_snap_time = GetCurrentTimestamp(); + } Assert(XLogRecPtrIsValid(t_thrd.proc->exrto_read_lsn)); } @@ -3112,8 +3209,8 @@ XLogRecPtr calculate_force_recycle_lsn_per_worker(StandbyReadMetaInfo* meta_info meta_info->batch_id, meta_info->redo_id, lsn_info_recycle_pos))); } LockBuffer(buffer, BUFFER_LOCK_SHARE); - extreme_rto_standby_read::LsnInfo lsn_info = (extreme_rto_standby_read::LsnInfo)(page + - extreme_rto_standby_read::LSN_INFO_HEAD_SIZE); + extreme_rto_standby_read::LsnInfo lsn_info = + (extreme_rto_standby_read::LsnInfo)(page + extreme_rto_standby_read::LSN_INFO_HEAD_SIZE); lsn_info_recycle_lsn = lsn_info->lsn[0]; UnlockReleaseBuffer(buffer); } @@ -3129,7 +3226,7 @@ void calculate_force_recycle_lsn(XLogRecPtr &recycle_lsn) for (uint32 i = 0; i < worker_nums; ++i) { PageRedoWorker* page_redo_worker = workers[i]; - if (page_redo_worker->role != REDO_PAGE_WORKER) { + if (page_redo_worker->role != REDO_PAGE_WORKER || (page_redo_worker->isUndoSpaceWorker)) { continue; } recycle_lsn_per_worker = calculate_force_recycle_lsn_per_worker(&page_redo_worker->standby_read_meta_info); @@ -3144,6 +3241,9 @@ void calculate_force_recycle_lsn(XLogRecPtr &recycle_lsn) static inline bool exceed_standby_max_query_time(TimestampTz start_time) { + if (start_time == 0) { + return false; + } return TimestampDifferenceExceeds(start_time, GetCurrentTimestamp(), g_instance.attr.attr_storage.standby_max_query_time * MSECS_PER_SEC); } @@ -3161,13 +3261,18 @@ void proc_array_get_oldeset_readlsn(XLogRecPtr recycle_lsn, XLogRecPtr &oldest_l int pg_proc_no = proc_array->pgprocnos[index]; PGPROC* pg_proc = g_instance.proc_base_all_procs[pg_proc_no]; PGXACT* pg_xact = &g_instance.proc_base_all_xacts[pg_proc_no]; - XLogRecPtr read_lsn = pg_proc->exrto_read_lsn; TransactionId pxmin = pg_xact->xmin; - - if (pg_proc->pid == 0 || !TransactionIdIsValid(pxmin) || XLogRecPtrIsInvalid(read_lsn)) { + XLogRecPtr read_lsn = pg_proc->exrto_min; + ereport( + DEBUG1, + (errmsg(EXRTOFORMAT("proc_array_get_oldeset_readlsn info, read_lsn: %08X/%08X ,xmin: %lu ,vacuum_flags: " + "%hhu ,pid: %lu"), + (uint32)(read_lsn >> UINT64_HALF), (uint32)read_lsn, pxmin, pg_xact->vacuumFlags, pg_proc->pid))); + + if (pg_proc->pid == 0 || XLogRecPtrIsInvalid(read_lsn)) { continue; } - + Assert(!(pg_xact->vacuumFlags & PROC_IN_VACUUM)); /* * Backend is doing logical decoding which manages xmin @@ -3187,6 +3292,13 @@ void proc_array_get_oldeset_readlsn(XLogRecPtr recycle_lsn, XLogRecPtr &oldest_l * wanted so ignore any errors. */ (void)SendProcSignal(pg_proc->pid, PROCSIG_RECOVERY_CONFLICT_SNAPSHOT, pg_proc->backendId); + ereport( + LOG, + (errmsg( + EXRTOFORMAT("read_lsn is less than recycle_lsn or query time exceed max_query_time while " + "get_oldeset_readlsn, read_lsn %lu, " + "recycle_lsn: %lu, exrto_gen_snap_time: %ld, current_time: %ld, thread id = %lu\n"), + read_lsn, recycle_lsn, pg_proc->exrto_gen_snap_time, GetCurrentTimestamp(), pg_proc->pid))); /* * Wait a little bit for it to die so that we avoid flooding * an unresponsive backend when system is heavily loaded. @@ -3255,7 +3367,26 @@ XLogRecPtr exrto_calculate_recycle_position(bool force_recyle) if (force_recyle) { calculate_force_recycle_lsn(recycle_lsn); } - + ereport(DEBUG1, (errmsg(EXRTOFORMAT("time information of calculate recycle position, current_time: %ld, snapshot " + "read_lsn: %08X/%08X, gen_snaptime:%ld"), + GetCurrentTimestamp(), + (uint32)(g_instance.comm_cxt.predo_cxt.exrto_snapshot->read_lsn >> UINT64_HALF), + (uint32)g_instance.comm_cxt.predo_cxt.exrto_snapshot->read_lsn, + g_instance.comm_cxt.predo_cxt.exrto_snapshot->gen_snap_time))); + + /* + * If there is no backend read threads, set read oldest lsn to snapshot lsn. + */ + ExrtoSnapshot exrto_snapshot = NULL; + exrto_snapshot = g_instance.comm_cxt.predo_cxt.exrto_snapshot; + (void)LWLockAcquire(ExrtoSnapshotLock, LW_SHARED); + if (XLByteEQ(exrto_snapshot->read_lsn, 0)) { + ereport(WARNING, (errmsg("could not get a valid snapshot with extreme rto"))); + } else { + oldest_lsn = exrto_snapshot->read_lsn; + oldest_xmin = exrto_snapshot->xmin; + } + LWLockRelease(ExrtoSnapshotLock); /* Loop checks to avoid conflicting queries that were not successfully canceled. */ do { RedoInterruptCallBack(); @@ -3263,25 +3394,8 @@ XLogRecPtr exrto_calculate_recycle_position(bool force_recyle) check_times++; } while (conflict && check_times < max_check_times); - /* - * If there is no backend read threads, set read oldest lsn to snapshot lsn. - */ - if (XLogRecPtrIsInvalid(oldest_lsn)) { - ExrtoSnapshot exrto_snapshot = NULL; - exrto_snapshot = &g_dispatcher->exrto_snapshot; - (void)LWLockAcquire(ExrtoSnapshotLock, LW_SHARED); - if (XLByteEQ(exrto_snapshot->read_lsn, 0)) { - ereport(WARNING, - (errmsg("could not get a valid snapshot with extreme rto"))); - } else { - oldest_lsn = exrto_snapshot->read_lsn; - oldest_xmin = exrto_snapshot->xmin; - } - - LWLockRelease(ExrtoSnapshotLock); - } recycle_lsn = rtl::max(recycle_lsn, oldest_lsn); - + ereport( LOG, (errmsg( @@ -3289,9 +3403,10 @@ XLogRecPtr exrto_calculate_recycle_position(bool force_recyle) "[exrto_recycle] calculate recycle position, oldestlsn: %08X/%08X, snapshot read_lsn: %08X/%08X, try " "recycle lsn: %08X/%08X"), (uint32)(oldest_lsn >> UINT64_HALF), (uint32)oldest_lsn, - (uint32)(g_dispatcher->exrto_snapshot.read_lsn >> UINT64_HALF), - (uint32)g_dispatcher->exrto_snapshot.read_lsn, (uint32)(recycle_lsn >> UINT64_HALF), (uint32)recycle_lsn))); - + (uint32)(g_instance.comm_cxt.predo_cxt.exrto_snapshot->read_lsn >> UINT64_HALF), + (uint32)g_instance.comm_cxt.predo_cxt.exrto_snapshot->read_lsn, + (uint32)(recycle_lsn >> UINT64_HALF), (uint32)recycle_lsn))); + return recycle_lsn; } @@ -3306,9 +3421,9 @@ TransactionId exrto_calculate_recycle_xmin_for_undo() /* * If there is no backend read threads, set read oldest lsn to snapshot lsn. */ - if (oldest_xmin == InvalidTransactionId) { + if ((oldest_xmin == InvalidTransactionId) && (g_dispatcher != NULL)) { ExrtoSnapshot exrto_snapshot = NULL; - exrto_snapshot = &g_dispatcher->exrto_snapshot; + exrto_snapshot = g_instance.comm_cxt.predo_cxt.exrto_snapshot; (void)LWLockAcquire(ExrtoSnapshotLock, LW_SHARED); if (XLByteEQ(exrto_snapshot->xmin, InvalidTransactionId)) { ereport( diff --git a/src/gausskernel/storage/access/transam/multi_redo_api.cpp b/src/gausskernel/storage/access/transam/multi_redo_api.cpp index 7b2b56444..5697de461 100644 --- a/src/gausskernel/storage/access/transam/multi_redo_api.cpp +++ b/src/gausskernel/storage/access/transam/multi_redo_api.cpp @@ -338,9 +338,8 @@ void CountXLogNumbers(XLogReaderState *record) record->readblocks); } else if (rm_id == RM_XACT_ID) { ColFileNode *xnodes = NULL; - bool compress = false; int nrels = 0; - XactGetRelFiles(record, &xnodes, &nrels, &compress); + XactGetRelFiles(record, &xnodes, &nrels); if (nrels > 0) { (void)pg_atomic_add_fetch_u64(&g_instance.comm_cxt.predo_cxt.xlogStatics[rm_id][info].extra_num, nrels); } diff --git a/src/gausskernel/storage/access/transam/multi_redo_settings.cpp b/src/gausskernel/storage/access/transam/multi_redo_settings.cpp index e26dba43a..d0007c8fc 100644 --- a/src/gausskernel/storage/access/transam/multi_redo_settings.cpp +++ b/src/gausskernel/storage/access/transam/multi_redo_settings.cpp @@ -56,7 +56,7 @@ void ConfigRecoveryParallelism() g_instance.attr.attr_storage.recovery_parse_workers, g_instance.attr.attr_storage.recovery_redo_workers_per_paser_worker, total_recovery_parallelism))); - g_supportHotStandby = false; + g_supportHotStandby = g_instance.attr.attr_storage.EnableHotStandby; SetConfigOption("recovery_parallelism", buf, PGC_POSTMASTER, PGC_S_OVERRIDE); } else if (g_instance.attr.attr_storage.max_recovery_parallelism > 1) { g_instance.comm_cxt.predo_cxt.redoType = PARALLEL_REDO; @@ -144,6 +144,9 @@ void ParseBindCpuInfo(RedoCpuBindControl *control) ptoken = TrimStr(strtok_r(attr, pdelimiter, &psave)); ptoken = pg_strtolower(ptoken); + if (ptoken == NULL) { + return; + } int bindNum = 0; if (strncmp("nobind", ptoken, strlen("nobind")) == 0) { diff --git a/src/gausskernel/storage/access/transam/ondemand_extreme_rto/page_redo.cpp b/src/gausskernel/storage/access/transam/ondemand_extreme_rto/page_redo.cpp index 56ea53b33..b93a19a0a 100644 --- a/src/gausskernel/storage/access/transam/ondemand_extreme_rto/page_redo.cpp +++ b/src/gausskernel/storage/access/transam/ondemand_extreme_rto/page_redo.cpp @@ -3046,8 +3046,9 @@ bool XactHasSegpageRelFiles(XLogReaderState *record) if (XLogRecGetRmid(record) != RM_XACT_ID) { return false; } - bool compress; - XactGetRelFiles(record, &xnodes, &nrels, &compress); + + bool compress = (bool)(XLogRecGetInfo(record) & XLR_REL_COMPRESS); + XactGetRelFiles(record, &xnodes, &nrels); for (int32 idx = 0; idx < nrels; idx++) { ColFileNode colFileNode; diff --git a/src/gausskernel/storage/access/transam/parallel_recovery/dispatcher.cpp b/src/gausskernel/storage/access/transam/parallel_recovery/dispatcher.cpp index 40fb6dc16..b8b574ace 100755 --- a/src/gausskernel/storage/access/transam/parallel_recovery/dispatcher.cpp +++ b/src/gausskernel/storage/access/transam/parallel_recovery/dispatcher.cpp @@ -503,8 +503,8 @@ static void StopRecoveryWorkers(int code, Datum arg) /* Run from the dispatcher thread. */ static void DestroyRecoveryWorkers() { + SpinLockAcquire(&(g_instance.comm_cxt.predo_cxt.destroy_lock)); if (g_dispatcher != NULL) { - SpinLockAcquire(&(g_instance.comm_cxt.predo_cxt.destroy_lock)); for (uint32 i = 0; i < g_dispatcher->totalWorkerCount; i++) DestroyPageRedoWorker(g_dispatcher->pageWorkers[i]); if (g_dispatcher->txnWorker != NULL) @@ -519,8 +519,8 @@ static void DestroyRecoveryWorkers() g_instance.comm_cxt.predo_cxt.parallelRedoCtx = NULL; } g_dispatcher = NULL; - SpinLockRelease(&(g_instance.comm_cxt.predo_cxt.destroy_lock)); } + SpinLockRelease(&(g_instance.comm_cxt.predo_cxt.destroy_lock)); } static bool RmgrRecordInfoValid(XLogReaderState *record, uint8 minInfo, uint8 maxInfo) @@ -1894,10 +1894,12 @@ uint32 GetStartupBufferPinWaitBufLen() */ void GetStartupBufferPinWaitBufId(int *bufids, uint32 len) { - for (uint32 i = 0; i < len - 1; i++) { - bufids[i] = g_dispatcher->pageWorkers[i]->bufferPinWaitBufId; + if (g_dispatcher != NULL) { + for (uint32 i = 0; i < len - 1; i++) { + bufids[i] = g_dispatcher->pageWorkers[i]->bufferPinWaitBufId; + } + bufids[len - 1] = g_instance.proc_base->startupBufferPinWaitBufId; } - bufids[len - 1] = g_instance.proc_base->startupBufferPinWaitBufId; } void GetReplayedRecPtrFromUndoWorkers(XLogRecPtr *readPtr, XLogRecPtr *endPtr) diff --git a/src/gausskernel/storage/access/transam/xact.cpp b/src/gausskernel/storage/access/transam/xact.cpp index 68e632abb..a6ff43cd4 100755 --- a/src/gausskernel/storage/access/transam/xact.cpp +++ b/src/gausskernel/storage/access/transam/xact.cpp @@ -47,6 +47,7 @@ #include "access/xlogutils.h" #include "access/multi_redo_api.h" #include "access/extreme_rto/standby_read/block_info_meta.h" +#include "access/extreme_rto/standby_read/standby_read_delay_ddl.h" #include "catalog/catalog.h" #include "catalog/namespace.h" #include "catalog/pg_authid.h" @@ -1539,8 +1540,9 @@ static TransactionId RecordTransactionCommit(void) if (useLocalXid || !IsPostmasterEnvironment || GTM_FREE_MODE) { #ifndef ENABLE_MULTIPLE_NODES /* For hot standby, set csn to commit in progress */ - CommitSeqNo csn = SetXact2CommitInProgress(xid, 0); - XLogInsertStandbyCSNCommitting(xid, csn, children, nchildren); + CommitSeqNo latestCsn = t_thrd.xact_cxt.ShmemVariableCache->nextCommitSeqNo; + XLogInsertStandbyCSNCommitting(xid, latestCsn, children, nchildren); + (void)SetXact2CommitInProgress(xid, 0); #else /* set commit CSN and update global CSN in gtm free mode. */ SetXact2CommitInProgress(xid, 0); @@ -7082,7 +7084,7 @@ static void xact_redo_forget_alloc_segs(TransactionId xid, TransactionId *subXid remainSegsLock.unLock(); } -static void xact_redo_log_drop_segs(_in_ ColFileNode *xnodes, _in_ int nrels, XLogRecPtr lsn) +void xact_redo_log_drop_segs(_in_ ColFileNode *xnodes, _in_ int nrels, XLogRecPtr lsn) { bool isNeedLogRemainSegs = IsNeedLogRemainSegs(lsn); if (!isNeedLogRemainSegs) { @@ -7173,7 +7175,7 @@ void push_unlink_rel_to_hashtbl(ColFileNode *xnodes, int nrels) /* * XLOG support routines */ -static void unlink_relfiles(_in_ ColFileNode *xnodes, _in_ int nrels) +void unlink_relfiles(_in_ ColFileNode *xnodes, _in_ int nrels) { ColMainFileNodesCreate(); @@ -7210,7 +7212,7 @@ static void unlink_relfiles(_in_ ColFileNode *xnodes, _in_ int nrels) /* * recycle exrto files when dropping table occurs. */ - if (IS_EXRTO_READ) { + if (RecoveryInProgress() && IS_EXRTO_READ) { RelFileNode block_meta_file = relFileNode; block_meta_file.spcNode = EXRTO_BLOCK_INFO_SPACE_OID; extreme_rto_standby_read::remove_one_block_info_file(block_meta_file); @@ -7243,6 +7245,32 @@ static void unlink_relfiles(_in_ ColFileNode *xnodes, _in_ int nrels) ColMainFileNodesDestroy(); } +void send_delay_invalid_message() +{ + if (t_thrd.page_redo_cxt.invalid_msg.valid) { + ProcessCommittedInvalidationMessages( + t_thrd.page_redo_cxt.invalid_msg.inval_msgs, t_thrd.page_redo_cxt.invalid_msg.nmsgs, + t_thrd.page_redo_cxt.invalid_msg.relcache_init_file_inval, t_thrd.page_redo_cxt.invalid_msg.db_id, + t_thrd.page_redo_cxt.invalid_msg.ts_id, t_thrd.page_redo_cxt.invalid_msg.lsn); + t_thrd.page_redo_cxt.invalid_msg.valid = false; + } +} + +void record_delay_invalid_message( + SharedInvalidationMessage* msgs, int nmsgs, bool relcache_init_file_inval, Oid dbid, Oid tsid, XLogRecPtr lsn) +{ + if (nmsgs <= 0) { + return; + } + t_thrd.page_redo_cxt.invalid_msg.inval_msgs = msgs; + t_thrd.page_redo_cxt.invalid_msg.nmsgs = nmsgs; + t_thrd.page_redo_cxt.invalid_msg.relcache_init_file_inval = relcache_init_file_inval; + t_thrd.page_redo_cxt.invalid_msg.db_id = dbid; + t_thrd.page_redo_cxt.invalid_msg.ts_id = tsid; + t_thrd.page_redo_cxt.invalid_msg.lsn = lsn; + t_thrd.page_redo_cxt.invalid_msg.valid = true; +} + /* * Before 9.0 this was a fairly short function, but now it performs many * actions for which the order of execution is critical. @@ -7310,7 +7338,7 @@ static void xact_redo_commit_internal(TransactionId xid, XLogRecPtr lsn, Transac } if (EnableGlobalSysCache()) { ProcessCommittedInvalidationMessages(inval_msgs, nmsgs, XactCompletionRelcacheInitFileInval(xinfo), - dbId, tsId); + dbId, tsId, lsn); } #endif } else { @@ -7370,8 +7398,13 @@ static void xact_redo_commit_internal(TransactionId xid, XLogRecPtr lsn, Transac * maintain the same order of invalidation then release locks as * occurs in CommitTransaction(). */ - ProcessCommittedInvalidationMessages(inval_msgs, nmsgs, XactCompletionRelcacheInitFileInval(xinfo), dbId, tsId); - + if (IS_EXRTO_READ) { + record_delay_invalid_message(inval_msgs, nmsgs, XactCompletionRelcacheInitFileInval(xinfo), dbId, + tsId, lsn); + } else { + ProcessCommittedInvalidationMessages(inval_msgs, nmsgs, XactCompletionRelcacheInitFileInval(xinfo), dbId, + tsId, lsn); + } /* * Release locks, if any. We do this for both two phase and normal one * phase transactions. In effect we are ignoring the prepare phase and @@ -7417,8 +7450,12 @@ static void xact_redo_commit_internal(TransactionId xid, XLogRecPtr lsn, Transac ColFileNodeCopy(&newColFileNodes[i], &colFileNodeRel[i]); } } - unlink_relfiles(newColFileNodes, nrels); - xact_redo_log_drop_segs(newColFileNodes, nrels, lsn); + if (IS_EXRTO_READ) { + update_delay_ddl_files(newColFileNodes, nrels, lsn); + } else { + unlink_relfiles(newColFileNodes, nrels); + xact_redo_log_drop_segs(newColFileNodes, nrels, lsn); + } if (unlikely((long)!compress)) { pfree(newColFileNodes); } @@ -7619,7 +7656,11 @@ static void xact_redo_abort(xl_xact_abort *xlrec, TransactionId xid, XLogRecPtr newColFileNodes[i].filenode.opt = 0; } } - unlink_relfiles(newColFileNodes, xlrec->nrels); + if (IS_EXRTO_READ) { + update_delay_ddl_files(newColFileNodes, xlrec->nrels, lsn); + } else { + unlink_relfiles(newColFileNodes, xlrec->nrels); + } xact_redo_log_drop_segs(newColFileNodes, xlrec->nrels, lsn); if (unlikely((long)!compress)) { pfree(newColFileNodes); @@ -7727,12 +7768,52 @@ void xact_redo(XLogReaderState *record) } } -void XactGetRelFiles(XLogReaderState *record, ColFileNode **xnodesPtr, int *nrelsPtr, bool *compress) +bool xact_has_invalid_msg_or_delete_file(XLogReaderState *record) +{ + Assert(XLogRecGetRmid(record) == RM_XACT_ID); + + uint8 info = (XLogRecGetInfo(record) & (~XLR_INFO_MASK)); + xl_xact_commit *commit = NULL; + xl_xact_abort *abort = NULL; + int msg_files = 0; + + switch (info) { + case XLOG_XACT_COMMIT_COMPACT: + case XLOG_XACT_PREPARE: + case XLOG_XACT_ASSIGNMENT: + break; + case XLOG_XACT_COMMIT: + commit = (xl_xact_commit *)XLogRecGetData(record); + msg_files = commit->nmsgs + commit->nrels; + break; + case XLOG_XACT_ABORT_WITH_XID: + case XLOG_XACT_ABORT: + abort = (xl_xact_abort *)XLogRecGetData(record); + msg_files = abort->nrels; + break; + case XLOG_XACT_COMMIT_PREPARED: + commit = &(((xl_xact_commit_prepared *)XLogRecGetData(record))->crec); + msg_files = commit->nmsgs + commit->nrels; + break; + case XLOG_XACT_ABORT_PREPARED: + abort = &(((xl_xact_abort_prepared *)XLogRecGetData(record))->arec); + msg_files = abort->nrels; + break; + default: + ereport(PANIC, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), + errmsg("xactWillRemoveRelFiles: unknown op code %u", (uint32)info))); + } + + return (msg_files > 0); +} + + +void XactGetRelFiles(XLogReaderState *record, ColFileNode **xnodesPtr, int *nrelsPtr) { Assert(XLogRecGetRmid(record) == RM_XACT_ID); uint8 info = (XLogRecGetInfo(record) & (~XLR_INFO_MASK)); - *compress = (bool)(XLogRecGetInfo(record) & XLR_REL_COMPRESS); + xl_xact_commit *commit = NULL; xl_xact_abort *abort = NULL; @@ -7781,13 +7862,12 @@ bool XactWillRemoveRelFiles(XLogReaderState *record) */ int nrels = 0; ColFileNode *xnodes = NULL; - bool compress = false; if (XLogRecGetRmid(record) != RM_XACT_ID) { return false; } - XactGetRelFiles(record, &xnodes, &nrels, &compress); + XactGetRelFiles(record, &xnodes, &nrels); return (nrels > 0); } @@ -7798,8 +7878,7 @@ bool xactWillRemoveRelFiles(XLogReaderState *record) ColFileNode *xnodes = NULL; Assert(XLogRecGetRmid(record) == RM_XACT_ID); - bool compress; - XactGetRelFiles(record, &xnodes, &nrels, &compress); + XactGetRelFiles(record, &xnodes, &nrels); return nrels > 0; } @@ -7807,8 +7886,9 @@ void xactApplyXLogDropRelation(XLogReaderState *record) { int nrels = 0; ColFileNode *xnodes = NULL; - bool compress; - XactGetRelFiles(record, &xnodes, &nrels, &compress); + + bool compress = (bool)(XLogRecGetInfo(record) & XLR_REL_COMPRESS); + XactGetRelFiles(record, &xnodes, &nrels); for (int i = 0; i < nrels; i++) { RelFileNodeBackend rbnode; ColFileNode node; diff --git a/src/gausskernel/storage/access/transam/xlog.cpp b/src/gausskernel/storage/access/transam/xlog.cpp index 66c6aba57..6d121a179 100755 --- a/src/gausskernel/storage/access/transam/xlog.cpp +++ b/src/gausskernel/storage/access/transam/xlog.cpp @@ -54,6 +54,7 @@ #include "access/hash.h" #include "access/xlogproc.h" #include "access/parallel_recovery/dispatcher.h" +#include "access/extreme_rto/page_redo.h" #include "commands/tablespace.h" #include "commands/matview.h" @@ -3157,37 +3158,11 @@ void UpdateMinRecoveryPoint(XLogRecPtr lsn, bool force) if (t_thrd.xlog_cxt.minRecoveryPoint == 0) { t_thrd.xlog_cxt.updateMinRecoveryPoint = false; } else if (force || XLByteLT(t_thrd.xlog_cxt.minRecoveryPoint, lsn)) { - /* use volatile pointer to prevent code rearrangement */ - volatile XLogCtlData *xlogctl = t_thrd.shemem_ptr_cxt.XLogCtl; - XLogRecPtr newMinRecoveryPoint; - - /* - * To avoid having to update the control file too often, we update it - * all the way to the last record being replayed, even though 'lsn' - * would suffice for correctness. This also allows the 'force' case - * to not need a valid 'lsn' value. - * - * Another important reason for doing it this way is that the passed - * 'lsn' value could be bogus, i.e., past the end of available WAL, if - * the caller got it from a corrupted heap page. Accepting such a - * value as the min recovery point would prevent us from coming up at - * all. Instead, we just log a warning and continue with recovery. - * (See also the comments about corrupt LSNs in XLogFlush.) - */ - SpinLockAcquire(&xlogctl->info_lck); - newMinRecoveryPoint = xlogctl->lastReplayedEndRecPtr; - SpinLockRelease(&xlogctl->info_lck); - - if (!force && XLByteLT(newMinRecoveryPoint, lsn) && !enable_heap_bcm_data_replication()) { - ereport(DEBUG1, (errmsg("xlog min recovery request %X/%X is past current point %X/%X", (uint32)(lsn >> 32), - (uint32)lsn, (uint32)(newMinRecoveryPoint >> 32), (uint32)newMinRecoveryPoint))); - } - /* update control file */ - if (XLByteLT(t_thrd.shemem_ptr_cxt.ControlFile->minRecoveryPoint, newMinRecoveryPoint)) { - t_thrd.shemem_ptr_cxt.ControlFile->minRecoveryPoint = newMinRecoveryPoint; + if (XLByteLT(t_thrd.shemem_ptr_cxt.ControlFile->minRecoveryPoint, lsn)) { + t_thrd.shemem_ptr_cxt.ControlFile->minRecoveryPoint = lsn; UpdateControlFile(); - t_thrd.xlog_cxt.minRecoveryPoint = newMinRecoveryPoint; + t_thrd.xlog_cxt.minRecoveryPoint = lsn; SetMinRecoverPointForStats(t_thrd.xlog_cxt.minRecoveryPoint); ereport(DEBUG1, (errmsg("updated min recovery point to %X/%X", (uint32)(t_thrd.xlog_cxt.minRecoveryPoint >> 32), @@ -5380,7 +5355,7 @@ static XLogRecord *ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, in * So err on the safe side and give up. */ if (!t_thrd.xlog_cxt.InArchiveRecovery && t_thrd.xlog_cxt.ArchiveRecoveryRequested && !fetching_ckpt) { - ProcTxnWorkLoad(false); + ProcTxnWorkLoad(true); volatile XLogCtlData *xlogctl = t_thrd.shemem_ptr_cxt.XLogCtl; XLogRecPtr newMinRecoveryPoint; ereport(DEBUG1, (errmsg_internal("reached end of WAL in pg_xlog, entering archive recovery"))); @@ -5435,7 +5410,7 @@ void UpdateMinrecoveryInAchive() { volatile XLogCtlData *xlogctl = t_thrd.shemem_ptr_cxt.XLogCtl; XLogRecPtr newMinRecoveryPoint; - + extreme_rto::PushToWorkerLsn(true); /* initialize minRecoveryPoint to this record */ LWLockAcquire(ControlFileLock, LW_EXCLUSIVE); t_thrd.shemem_ptr_cxt.ControlFile->state = DB_IN_ARCHIVE_RECOVERY; @@ -5453,7 +5428,7 @@ void UpdateMinrecoveryInAchive() UpdateControlFile(); LWLockRelease(ControlFileLock); MultiRedoUpdateMinRecovery(t_thrd.xlog_cxt.minRecoveryPoint); - + t_thrd.xlog_cxt.updateMinRecoveryPoint = true; // for extreme rto xlog page read worker, no need send lsn forwarder ereport(LOG, (errmsg("update minrecovery point to %X/%X in archive recovery", (uint32)(t_thrd.xlog_cxt.minRecoveryPoint >> 32), (uint32)(t_thrd.xlog_cxt.minRecoveryPoint)))); @@ -8832,6 +8807,27 @@ static inline void UpdateTermFromXLog(uint32 xlTerm) } } +void init_extreme_rto_standby_read_first_snapshot(const XLogRecPtr checkpoint_loc) +{ + if (!IsExtremeRedo()) { + return; + } + + if (!g_instance.attr.attr_storage.EnableHotStandby) { + return; + } + + g_instance.comm_cxt.predo_cxt.exrto_snapshot->gen_snap_time = 0; + g_instance.comm_cxt.predo_cxt.exrto_snapshot->read_lsn = checkpoint_loc; + g_instance.comm_cxt.predo_cxt.exrto_snapshot->snapshot_csn = t_thrd.xact_cxt.ShmemVariableCache->nextCommitSeqNo; + g_instance.comm_cxt.predo_cxt.exrto_snapshot->xmin = t_thrd.xact_cxt.ShmemVariableCache->latestCompletedXid; + g_instance.comm_cxt.predo_cxt.exrto_snapshot->xmax = t_thrd.xact_cxt.ShmemVariableCache->latestCompletedXid; + if (TransactionIdIsValid(t_thrd.xact_cxt.ShmemVariableCache->standbyXmin) && + t_thrd.xact_cxt.ShmemVariableCache->standbyXmin <= t_thrd.xact_cxt.ShmemVariableCache->latestCompletedXid) { + g_instance.comm_cxt.predo_cxt.exrto_snapshot->xmin = t_thrd.xact_cxt.ShmemVariableCache->standbyXmin; + } +} + /* * This must be called ONCE during postmaster or standalone-backend startup */ @@ -9914,6 +9910,8 @@ void StartupXLOG(void) (uint32)t_thrd.shemem_ptr_cxt.ControlFile->backupStartPoint, t_thrd.shemem_ptr_cxt.ControlFile->backupEndRequired ? "TRUE" : "FALSE"))); + + init_extreme_rto_standby_read_first_snapshot(checkPoint.redo); pg_atomic_write_u32(&t_thrd.walreceiverfuncs_cxt.WalRcv->rcvDoneFromShareStorage, false); // Allow read-only connections immediately if we're consistent already. CheckRecoveryConsistency(); @@ -10437,7 +10435,9 @@ void StartupXLOG(void) if (IS_EXRTO_READ) { /* we are going to be master, we need to recycle residual_undo_file again */ + (void)LWLockAcquire(ExrtoRecycleResidualUndoLock, LW_EXCLUSIVE); g_instance.undo_cxt.is_exrto_residual_undo_file_recycled = false; + LWLockRelease(ExrtoRecycleResidualUndoLock); } LocalSetXLogInsertAllowed(); @@ -12766,7 +12766,8 @@ bool CreateRestartPoint(int flags) return false; } } - + (void)LWLockAcquire(RedoTruncateLock, LW_SHARED); + LWLockRelease(RedoTruncateLock); /* * Update pg_control, using current time. Check that it still shows * IN_ARCHIVE_RECOVERY state and an older checkpoint, else do nothing; @@ -13199,11 +13200,13 @@ static void KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo, XLogRecPtr curIns } } - if (IS_EXRTO_READ) { + if (RecoveryInProgress() && IS_EXRTO_READ) { XLogRecPtr recycle_recptr = pg_atomic_read_u64(&g_instance.comm_cxt.predo_cxt.global_recycle_lsn); XLogSegNo recyle_segno; XLByteToSeg(recycle_recptr, recyle_segno); - if (recyle_segno < segno && recyle_segno > 0) { + if (recyle_segno == 0) { + segno = 1; + } else if (recyle_segno < segno) { segno = recyle_segno; } } @@ -13409,6 +13412,14 @@ bool IsCheckPoint(const XLogRecParseState *parseState) return rmid == RM_XLOG_ID && (info == XLOG_CHECKPOINT_SHUTDOWN || info == XLOG_CHECKPOINT_ONLINE); } +bool is_backup_end(const XLogRecParseState *parse_state) +{ + uint8 info = parse_state->blockparse.blockhead.xl_info & (~XLR_INFO_MASK); + RmgrId rmid = parse_state->blockparse.blockhead.xl_rmid; + + return rmid == RM_XLOG_ID && (info == XLOG_BACKUP_END); +} + bool HasTimelineUpdate(XLogReaderState *record) { uint8 info; @@ -16364,6 +16375,9 @@ static bool read_tablespace_map(List **tablespaces) /* * Error context callback for errors occurring during rm_redo(). */ void rm_redo_error_callback(void *arg) { + if (arg == NULL) { + return; + } XLogReaderState *record = (XLogReaderState *)arg; StringInfoData buf; diff --git a/src/gausskernel/storage/access/transam/xlogutils.cpp b/src/gausskernel/storage/access/transam/xlogutils.cpp index c5db30a85..7bd8aee2e 100644 --- a/src/gausskernel/storage/access/transam/xlogutils.cpp +++ b/src/gausskernel/storage/access/transam/xlogutils.cpp @@ -1380,6 +1380,13 @@ void XlogDropRowReation(RelFileNode rnode) rbnode.node = rnode; rbnode.backend = InvalidBackendId; smgrclosenode(rbnode); + if (IS_EXRTO_READ) { + RelFileNodeBackend standbyReadRnode; + standbyReadRnode.node = rnode; + standbyReadRnode.node.spcNode = EXRTO_BLOCK_INFO_SPACE_OID; + standbyReadRnode.backend = InvalidBackendId; + smgrclosenode(standbyReadRnode); + } } void XLogForgetDDLRedo(XLogRecParseState *redoblockstate) @@ -1437,6 +1444,10 @@ void XLogDropSpaceShrink(XLogRecParseState *redoblockstate) */ void XLogDropRelation(const RelFileNode &rnode, ForkNumber forknum) { + if (AmErosRecyclerProcess()) { + return; + } + forget_invalid_pages(rnode, forknum, 0, false); /* clear relfilenode match entry of recovery thread hashtbl */ @@ -1514,6 +1525,10 @@ void XLogDropDatabase(Oid dbid) forget_invalid_pages_batch(InvalidOid, dbid); + if (AmErosRecyclerProcess()) { + return; + } + /* clear dbNode match entry of recovery thread hashtbl */ if (IsExtremeRedo()) { ExtremeBatchClearRecoveryThreadHashTbl(InvalidOid, dbid); @@ -1954,4 +1969,3 @@ XLogRecParseState *multixact_redo_parse_to_block(XLogReaderState *record, uint32 } return recordstatehead; } - diff --git a/src/gausskernel/storage/access/ubtree/ubtxlog.cpp b/src/gausskernel/storage/access/ubtree/ubtxlog.cpp index 798549cc2..78fedf888 100644 --- a/src/gausskernel/storage/access/ubtree/ubtxlog.cpp +++ b/src/gausskernel/storage/access/ubtree/ubtxlog.cpp @@ -970,7 +970,7 @@ static void UBTreeXlogReusePage(XLogReaderState *record) RelFileNode tmp_node; RelFileNodeCopy(tmp_node, xlrec->node, XLogRecGetBucketId(record)); - if (InHotStandby && g_supportHotStandby) { + if (InHotStandby && g_supportHotStandby && !IS_EXRTO_READ) { ResolveRecoveryConflictWithSnapshot(xlrec->latestRemovedXid, tmp_node, lsn); } } @@ -1000,7 +1000,7 @@ static void UBTreeXlogPrunePage(XLogReaderState* record) /* Caller specified a bogus block_id */ ereport(PANIC, (errmsg("failed to locate backup block with ID %d", 0))); } - if (InHotStandby && TransactionIdIsValid(xlrec->latestRemovedXid)) + if (InHotStandby && TransactionIdIsValid(xlrec->latestRemovedXid) && !IS_EXRTO_READ) ResolveRecoveryConflictWithSnapshot(xlrec->latestRemovedXid, rnode, lsn); if (XLogReadBufferForRedo(record, 0, &buffer) == BLK_NEEDS_REDO) { diff --git a/src/gausskernel/storage/access/ustore/knl_uextremeredo.cpp b/src/gausskernel/storage/access/ustore/knl_uextremeredo.cpp index 9149870fe..5797e42a4 100644 --- a/src/gausskernel/storage/access/ustore/knl_uextremeredo.cpp +++ b/src/gausskernel/storage/access/ustore/knl_uextremeredo.cpp @@ -1247,17 +1247,6 @@ void UHeapXlogCleanOperatorPage(RedoBufferInfo *buffer, void *recorddata, void * uint16 *nfixed = &tmpfixed; char *unused = (char *)xlrec + SizeOfUHeapClean; - /* - * We're about to remove tuples. In Hot Standby mode, ensure that there's - * no queries running for which the removed tuples are still visible. - * - * Not all UHEAP_CLEAN records remove tuples with xids, so we only want to - * conflict on the records that cause MVCC failures for user queries. If - * latestRemovedXid is invalid, skip conflict processing. - */ - if (InHotStandby && TransactionIdIsValid(xlrec->latestRemovedXid)) - ResolveRecoveryConflictWithSnapshot(xlrec->latestRemovedXid, buffer->blockinfo.rnode, buffer->lsn); - /* Update all item pointers per the record, and repair fragmentation */ if (xlrec->flags & XLZ_CLEAN_CONTAINS_OFFSET) { targetOffnum = (OffsetNumber *)((char *)xlrec + SizeOfUHeapClean); @@ -2011,12 +2000,13 @@ static void RedoUndoDiscardBlock(XLogBlockHead *blockhead, XLogBlockUndoParse *b if (zone->GetLSN() < lsn) { zone->LockUndoZone(); Assert(blockdatarec->undoDiscardParse.startSlot == zone->GetRecycleTSlotPtr()); - if (IS_EXRTO_READ && (!g_instance.undo_cxt.is_exrto_residual_undo_file_recycled)) { - zone->set_recycle_tslot_ptr_exrto(endSlot); - } zone->SetRecycleTSlotPtr(endSlot); zone->SetDiscardURecPtr(endUndoPtr); zone->SetForceDiscardURecPtr(endUndoPtr); + if (!IS_EXRTO_READ) { + zone->set_discard_urec_ptr_exrto(endUndoPtr); + zone->set_force_discard_urec_ptr_exrto(endUndoPtr); + } zone->SetRecycleXid(recycledXid); zone->MarkDirty(); zone->SetLSN(lsn); @@ -2042,10 +2032,16 @@ static void RedoUndoUnlinkBlock(XLogBlockHead *blockhead, XLogBlockUndoParse *bl zoneId, usp->LSN(), unlinkLsn, head, newHead))); if (usp->LSN() < unlinkLsn) { - zone->ForgetUndoBuffer(head, newHead, UNDO_DB_OID); + /* + * before hot_standby mode, we don,t know we will be primary or standby, + * so before hot standby we better do unlinklog. + */ + if (!IS_EXRTO_READ) { + zone->ForgetUndoBuffer(head, newHead, UNDO_DB_OID); + } usp->LockSpace(); usp->MarkDirty(); - if (IS_EXRTO_STANDBY_READ) { + if (IS_EXRTO_READ) { usp->SetHead(newHead); } else { usp->UnlinkUndoLog(zoneId, newHead, UNDO_DB_OID); @@ -2072,10 +2068,16 @@ static void RedoSlotUnlinkBlock(XLogBlockHead *blockhead, XLogBlockUndoParse *bl zoneId, usp->LSN(), unlinkLsn, head, newHead))); if (usp->LSN() < unlinkLsn) { - zone->ForgetUndoBuffer(head, newHead, UNDO_SLOT_DB_OID); + /* + * before hot_standby mode, we don,t know we will be primary or standby, + * so before hot standby we better do unlinklog. + */ + if (!IS_EXRTO_READ) { + zone->ForgetUndoBuffer(head, newHead, UNDO_SLOT_DB_OID); + } usp->LockSpace(); usp->MarkDirty(); - if (IS_EXRTO_STANDBY_READ) { + if (IS_EXRTO_READ) { usp->SetHead(newHead); } else { usp->UnlinkUndoLog(zoneId, newHead, UNDO_SLOT_DB_OID); diff --git a/src/gausskernel/storage/access/ustore/knl_utuptoaster.cpp b/src/gausskernel/storage/access/ustore/knl_utuptoaster.cpp index 8240de04d..2bb1448e6 100644 --- a/src/gausskernel/storage/access/ustore/knl_utuptoaster.cpp +++ b/src/gausskernel/storage/access/ustore/knl_utuptoaster.cpp @@ -977,7 +977,7 @@ struct varlena *UHeapInternalToastFetchDatum(struct varatt_external toastPointer */ nextidx = 0; - toastscan = systable_beginscan_ordered(toastrel, toastidx, SnapshotToast, 1, &toastkey); + toastscan = systable_beginscan_ordered(toastrel, toastidx, get_toast_snapshot(), 1, &toastkey); while (UHeapSysIndexGetnextSlot(toastscan, ForwardScanDirection, slot)) { /* * Have a chunk, extract the sequence number and the data @@ -1163,7 +1163,7 @@ struct varlena *UHeapInternalToastFetchDatumSlice(struct varatt_external toastPo * The index is on (valueid, chunkidx) so they will come in order */ nextidx = startchunk; - toastscan = systable_beginscan_ordered(toastrel, toastidx, SnapshotToast, nscankeys, toastkey); + toastscan = systable_beginscan_ordered(toastrel, toastidx, get_toast_snapshot(), nscankeys, toastkey); while (UHeapSysIndexGetnextSlot(toastscan, ForwardScanDirection, slot)) { /* * Have a chunk, extract the sequence number and the data diff --git a/src/gausskernel/storage/access/ustore/knl_uvisibility.cpp b/src/gausskernel/storage/access/ustore/knl_uvisibility.cpp index 48beaf0e1..a12abb556 100644 --- a/src/gausskernel/storage/access/ustore/knl_uvisibility.cpp +++ b/src/gausskernel/storage/access/ustore/knl_uvisibility.cpp @@ -249,6 +249,10 @@ bool UHeapTupleSatisfiesVisibility(UHeapTuple uhtup, Snapshot snapshot, Buffer b } uint64 globalFrozenXid = isFlashBack ? pg_atomic_read_u64(&g_instance.undo_cxt.globalRecycleXid) : pg_atomic_read_u64(&g_instance.undo_cxt.globalFrozenXid); + if (pm_state_is_hot_standby()) { + /* in hot standby mode, if globalRecycleXid advance during query, it may cause data inconsistency */ + globalFrozenXid = 0; + } if (TransactionIdIsValid(tdinfo.xid) && TransactionIdPrecedes(tdinfo.xid, globalFrozenXid)) { /* The slot is old enough that we can treat it as frozen. */ tdinfo.td_slot = UHEAPTUP_SLOT_FROZEN; @@ -813,6 +817,10 @@ bool UHeapTupleFetch(Relation rel, Buffer buffer, OffsetNumber offnum, Snapshot uint64 oldestRecycleXidHavingUndo = pg_atomic_read_u64(&g_instance.undo_cxt.globalRecycleXid); uint64 oldestXidHavingUndo = (isFlashBack || isLogical) ? oldestRecycleXidHavingUndo : pg_atomic_read_u64(&g_instance.undo_cxt.globalFrozenXid); + if (pm_state_is_hot_standby()) { + /* in hot standby mode, if globalRecycleXid advance during query, it may cause data inconsistency */ + oldestXidHavingUndo = 0; + } if (TransactionIdIsValid(tdinfo.xid) && TransactionIdPrecedes(tdinfo.xid, oldestXidHavingUndo)) { if (TransactionIdOlderThanAllUndo(tdinfo.xid)) { isFrozen = true; diff --git a/src/gausskernel/storage/access/ustore/undo/knl_uundoapi.cpp b/src/gausskernel/storage/access/ustore/undo/knl_uundoapi.cpp index 87acf77e5..4cbad291a 100644 --- a/src/gausskernel/storage/access/ustore/undo/knl_uundoapi.cpp +++ b/src/gausskernel/storage/access/ustore/undo/knl_uundoapi.cpp @@ -646,6 +646,7 @@ void RecoveryUndoSystemMeta(void) /* Close fd. */ close(fd); + exrto_recycle_residual_undo_file("recovery_meta"); ereport(LOG, (errcode(ERRCODE_DATA_EXCEPTION), errmsg("recovery_meta: undo recovery finish."))); #endif } diff --git a/src/gausskernel/storage/access/ustore/undo/knl_uundorecycle.cpp b/src/gausskernel/storage/access/ustore/undo/knl_uundorecycle.cpp index 44debc23e..b6201db3e 100755 --- a/src/gausskernel/storage/access/ustore/undo/knl_uundorecycle.cpp +++ b/src/gausskernel/storage/access/ustore/undo/knl_uundorecycle.cpp @@ -370,7 +370,7 @@ bool RecycleUndoSpace(UndoZone *zone, TransactionId recycleXmin, TransactionId f if (undoRecycled) { Assert(TransactionIdIsValid(recycleXid) && (zone->GetRecycleXid() < recycleXid)); zone->LockUndoZone(); - if (!zone->CheckRecycle(startUndoPtr, endUndoPtr)) { + if (!zone->CheckRecycle(startUndoPtr, endUndoPtr, false)) { ereport(PANIC, (errmodule(MOD_UNDO), errmsg(UNDOFORMAT("zone %d recycle start %lu >= recycle end %lu."), zone->GetZoneId(), startUndoPtr, endUndoPtr))); } @@ -536,7 +536,7 @@ void exrto_standby_release_space(UndoZone *zone, TransactionId recycle_xid, Undo UndoRecPtr oldest_end_undo_ptr = end_undo_ptr; Assert(TransactionIdIsValid(recycle_xid) && (zone->get_recycle_xid_exrto() < recycle_xid)); zone->LockUndoZone(); - if (!zone->CheckRecycle(start_undo_ptr, end_undo_ptr)) { + if (!zone->CheckRecycle(start_undo_ptr, end_undo_ptr, true)) { ereport(PANIC, (errmodule(MOD_UNDO), errmsg(UNDOFORMAT("zone %d recycle start %lu >= recycle end %lu."), zone->GetZoneId(), start_undo_ptr, end_undo_ptr))); @@ -549,7 +549,11 @@ void exrto_standby_release_space(UndoZone *zone, TransactionId recycle_xid, Undo } zone->set_discard_urec_ptr_exrto(oldest_end_undo_ptr); } - + + ereport(DEBUG1, (errmodule(MOD_STANDBY_READ), + errmsg("exrto_standby_release_space: zone %d recycle_xid %lu recycle start " + "%lu recycle end %lu recycle_tslot %lu.", + zone->GetZoneId(), recycle_xid, start_undo_ptr, end_undo_ptr, recycle_exrto))); zone->set_recycle_xid_exrto(recycle_xid); zone->set_force_discard_urec_ptr_exrto(end_undo_ptr); zone->set_recycle_tslot_ptr_exrto(recycle_exrto); @@ -644,17 +648,23 @@ bool exrto_standby_recycle_undo_zone() } /* recycle residual_undo_file which may be leftover by exrto read in standby */ -void exrto_recycle_residual_undo_file() +void exrto_recycle_residual_undo_file(char *FuncName) { uint32 idx = 0; uint64 record_file_cnt = 0; uint64 slot_file_cnt = 0; + (void)LWLockAcquire(ExrtoRecycleResidualUndoLock, LW_EXCLUSIVE); if (g_instance.undo_cxt.is_exrto_residual_undo_file_recycled) { + LWLockRelease(ExrtoRecycleResidualUndoLock); + ereport(LOG, (errmodule(MOD_UNDO), + errmsg(UNDOFORMAT("exrto_recycle_residual_undo_file skip, FuncName:%s."), FuncName))); return; } + g_instance.undo_cxt.is_exrto_residual_undo_file_recycled = true; + LWLockRelease(ExrtoRecycleResidualUndoLock); ereport(LOG, (errmodule(MOD_UNDO), - errmsg(UNDOFORMAT("exrto_recycle_residual_undo_file begin uZoneCount is %u."), - g_instance.undo_cxt.uZoneCount))); + errmsg(UNDOFORMAT("exrto_recycle_residual_undo_file begin uZoneCount is %u, FuncName:%s."), + g_instance.undo_cxt.uZoneCount, FuncName))); if (g_instance.undo_cxt.uZoneCount == 0 || g_instance.undo_cxt.uZones == NULL) { g_instance.undo_cxt.is_exrto_residual_undo_file_recycled = true; ereport(LOG, (errmodule(MOD_UNDO), @@ -673,7 +683,6 @@ void exrto_recycle_residual_undo_file() ereport(LOG, (errmodule(MOD_UNDO), errmsg(UNDOFORMAT("exrto_recycle_residual_undo_file release record_file_cnt:%lu, " "slot_file_cnt:%lu."), record_file_cnt, slot_file_cnt))); - g_instance.undo_cxt.is_exrto_residual_undo_file_recycled = true; } void recycle_wait(bool recycled, uint64 *non_recycled) @@ -799,6 +808,7 @@ void UndoRecycleMain() pg_usleep(10000000L); ereport(LOG, (errmodule(MOD_UNDO), errmsg(UNDOFORMAT("sleep 10s, ensure the snapcapturer can give the undorecyclemain a valid recycleXmin.")))); + exrto_recycle_residual_undo_file("recycle_main"); while (true) { if (t_thrd.undorecycler_cxt.got_SIGHUP) { t_thrd.undorecycler_cxt.got_SIGHUP = false; @@ -807,7 +817,6 @@ void UndoRecycleMain() if (t_thrd.undorecycler_cxt.shutdown_requested) { ShutDownRecycle(recycleMaxXIDs); } - exrto_recycle_residual_undo_file(); if (!RecoveryInProgress()) { TransactionId recycleXmin = InvalidTransactionId; TransactionId oldestXmin = GetOldestXminForUndo(&recycleXmin); diff --git a/src/gausskernel/storage/access/ustore/undo/knl_uundozone.cpp b/src/gausskernel/storage/access/ustore/undo/knl_uundozone.cpp index 7ac14bfd4..6ae2a8700 100644 --- a/src/gausskernel/storage/access/ustore/undo/knl_uundozone.cpp +++ b/src/gausskernel/storage/access/ustore/undo/knl_uundozone.cpp @@ -81,25 +81,28 @@ bool UndoZone::CheckNeedSwitch(UndoRecordSize size) return false; } -bool UndoZone::CheckRecycle(UndoRecPtr starturp, UndoRecPtr endurp) +bool UndoZone::CheckRecycle(UndoRecPtr starturp, UndoRecPtr endurp, bool isexrto) { int startZid = UNDO_PTR_GET_ZONE_ID(starturp); int endZid = UNDO_PTR_GET_ZONE_ID(endurp); UndoLogOffset start = UNDO_PTR_GET_OFFSET(starturp); UndoLogOffset end = UNDO_PTR_GET_OFFSET(endurp); - UndoLogOffset force_discard_urec_ptr; - if (IS_EXRTO_STANDBY_READ) { - force_discard_urec_ptr = force_discard_urec_ptr_exrto; - } else { - force_discard_urec_ptr = forceDiscardURecPtr_; - } - Assert(start == force_discard_urec_ptr); WHITEBOX_TEST_STUB(UNDO_CHECK_RECYCLE_FAILED, WhiteboxDefaultErrorEmit); - - if ((startZid == endZid) && (force_discard_urec_ptr <= insertURecPtr_) && (end <= insertURecPtr_) - && (start < end)) { - return true; + if (isexrto) { + if ((startZid == endZid) && (forceDiscardURecPtr_ <= insertURecPtr_) && (end <= insertURecPtr_) && + (start < end)) { + return true; + } + } else { + if ((startZid == endZid) && (forceDiscardURecPtr_ <= insertURecPtr_) && (end <= insertURecPtr_) && + (start < end) && (start == forceDiscardURecPtr_)) { + return true; + } } + ereport(WARNING, (errmodule(MOD_UNDO), + errmsg(UNDOFORMAT("check_recycle: zone:%d, startZid:%d, endZid:%d, start:%lu, end:%lu, " + "forceDiscardURecPtr_:%lu, insertURecPtr_:%lu."), + zid_, startZid, endZid, start, end, forceDiscardURecPtr_, insertURecPtr_))); return false; } @@ -312,9 +315,14 @@ void UndoZone::ReleaseSpace(UndoRecPtr starturp, UndoRecPtr endurp, int *forceRe /* Release undo space from starturp to endurp and advance discard. */ uint64 UndoZone::release_residual_record_space() { - undoSpace_.LockSpace(); UndoLogOffset unlink_start = undoSpace_.find_oldest_offset(zid_, UNDO_DB_OID); UndoLogOffset unlink_end = undoSpace_.Head(); + uint64 start_segno = unlink_start / UNDO_LOG_SEGMENT_SIZE; + uint64 end_segno = unlink_end / UNDO_LOG_SEGMENT_SIZE; + ereport(DEBUG1, (errmodule(MOD_STANDBY_READ), + errmsg("release_residual_record_space start_segno:%lu end_segno:%lu.", start_segno, end_segno))); + ForgetUndoBuffer(start_segno * UNDO_LOG_SEGMENT_SIZE, end_segno * UNDO_LOG_SEGMENT_SIZE, UNDO_DB_OID); + undoSpace_.LockSpace(); undoSpace_.unlink_residual_log(zid_, unlink_start, unlink_end, UNDO_DB_OID); undoSpace_.UnlockSpace(); if (unlink_start > unlink_end) { @@ -367,9 +375,12 @@ void UndoZone::ReleaseSlotSpace(UndoRecPtr startSlotPtr, UndoRecPtr endSlotPtr, /* Release slot space from starturp to endurp and advance discard. */ uint64 UndoZone::release_residual_slot_space() { - slotSpace_.LockSpace(); UndoLogOffset unlink_start = slotSpace_.find_oldest_offset(zid_, UNDO_SLOT_DB_OID); UndoLogOffset unlink_end = slotSpace_.Head(); + uint64 start_segno = unlink_start / UNDO_LOG_SEGMENT_SIZE; + uint64 end_segno = unlink_end / UNDO_LOG_SEGMENT_SIZE; + ForgetUndoBuffer(start_segno * UNDO_LOG_SEGMENT_SIZE, end_segno * UNDO_LOG_SEGMENT_SIZE, UNDO_SLOT_DB_OID); + slotSpace_.LockSpace(); slotSpace_.unlink_residual_log(zid_, unlink_start, unlink_end, UNDO_SLOT_DB_OID); slotSpace_.UnlockSpace(); if (unlink_start > unlink_end) { diff --git a/src/gausskernel/storage/buffer/bufmgr.cpp b/src/gausskernel/storage/buffer/bufmgr.cpp index c52978bda..a84d2bcb8 100644 --- a/src/gausskernel/storage/buffer/bufmgr.cpp +++ b/src/gausskernel/storage/buffer/bufmgr.cpp @@ -1628,6 +1628,47 @@ Buffer ReadBuffer(Relation reln, BlockNumber block_num) return ReadBufferExtended(reln, MAIN_FORKNUM, block_num, RBM_NORMAL, NULL); } +Buffer buffer_read_extended_internal(Relation reln, ForkNumber fork_num, BlockNumber block_num, ReadBufferMode mode, + BufferAccessStrategy strategy) +{ + bool hit = false; + Buffer buf; + + if (block_num == P_NEW) { + STORAGE_SPACE_OPERATION(reln, BLCKSZ); + } + + /* Open it at the smgr level */ + RelationOpenSmgr(reln); + + /* + * * Test for a temporary relation that belongs to some other session. + */ + if (RELATION_IS_OTHER_TEMP(reln) && fork_num <= INIT_FORKNUM) + /* + * We would be likely to get wrong data since we have no visibility into the owning session's local buffers. + */ + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot access temporary tables of other sessions"))); + + /* + * Read the buffer. + */ + pgstat_count_buffer_read(reln); + pgstatCountBlocksFetched4SessionLevel(); + + if (RelationisEncryptEnable(reln)) { + reln->rd_smgr->encrypt = true; + } + buf = + ReadBuffer_common(reln->rd_smgr, reln->rd_rel->relpersistence, fork_num, block_num, mode, strategy, &hit, NULL); + if (hit) { + /* Update pgstat counters to reflect a cache hit */ + pgstat_count_buffer_hit(reln); + } + return buf; +} + /* * ReadBufferExtended -- returns a buffer containing the requested * block of the requested relation. If the blknum @@ -1673,45 +1714,12 @@ Buffer ReadBuffer(Relation reln, BlockNumber block_num) Buffer ReadBufferExtended(Relation reln, ForkNumber fork_num, BlockNumber block_num, ReadBufferMode mode, BufferAccessStrategy strategy) { - if (IsDefaultExtremeRtoMode() && IsExtremeRtoRunning() && !AmPageRedoWorker()) { + if (IsDefaultExtremeRtoMode() && + (!RecoveryInProgress() || !IsExtremeRtoRunning() || !is_exrto_standby_read_worker())) { + return buffer_read_extended_internal(reln, fork_num, block_num, mode, strategy); + } else { return standby_read_buf(reln, fork_num, block_num, mode, strategy); } - - bool hit = false; - Buffer buf; - - if (block_num == P_NEW) { - STORAGE_SPACE_OPERATION(reln, BLCKSZ); - } - - /* Open it at the smgr level if not already done */ - RelationOpenSmgr(reln); - - /* - * Reject attempts to read non-local temporary relations; we would be - * likely to get wrong data since we have no visibility into the owning - * session's local buffers. - */ - if (RELATION_IS_OTHER_TEMP(reln) && fork_num <= INIT_FORKNUM) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot access temporary tables of other sessions"))); - - /* - * Read the buffer, and update pgstat counters to reflect a cache hit or - * miss. - */ - pgstat_count_buffer_read(reln); - pgstatCountBlocksFetched4SessionLevel(); - - if (RelationisEncryptEnable(reln)) { - reln->rd_smgr->encrypt = true; - } - buf = ReadBuffer_common(reln->rd_smgr, reln->rd_rel->relpersistence, fork_num, - block_num, mode, strategy, &hit, NULL); - if (hit) { - pgstat_count_buffer_hit(reln); - } - return buf; } /* @@ -5424,31 +5432,6 @@ void DropDatabaseBuffers(Oid dbid) gstrace_exit(GS_TRC_ID_DropDatabaseBuffers); } -void buffer_drop_exrto_standby_read_buffers() -{ - int i = 0; - ereport(LOG, (errmsg("buffer_drop_exrto_standby_read_buffers: start to drop buffers."))); - while (i < TOTAL_BUFFER_NUM) { - BufferDesc *buf_desc = GetBufferDescriptor(i); - uint32 buf_state; - /* - * Some safe unlocked checks can be done to reduce the number of cycle. - */ - if (!IS_EXRTO_RELFILENODE(buf_desc->tag.rnode)) { - i++; - continue; - } - - buf_state = LockBufHdr(buf_desc); - if (IS_EXRTO_RELFILENODE(buf_desc->tag.rnode)) { - InvalidateBuffer(buf_desc); /* with buffer head lock released */ - } else { - UnlockBufHdr(buf_desc, buf_state); - } - i++; - } -} - /* ----------------------------------------------------------------- * PrintBufferDescs * @@ -5718,11 +5701,6 @@ void MarkBufferDirtyHint(Buffer buffer, bool buffer_std) Assert(GetPrivateRefCount(buffer) > 0); - // temp buf just for old page version, could not write to disk - if (pg_atomic_read_u32(&buf_desc->state) & BM_IS_TMP_BUF) { - return; - } - /* here, either share or exclusive lock is OK */ if (!LWLockHeldByMe(buf_desc->content_lock)) ereport(PANIC, (errcode(ERRCODE_INVALID_BUFFER), @@ -5744,6 +5722,11 @@ void MarkBufferDirtyHint(Buffer buffer, bool buffer_std) bool delayChkpt = false; uint32 buf_state; uint32 old_buf_state; + buf_state = pg_atomic_read_u32(&buf_desc->state); + // temp buf just for old page version, could not write to disk + if (IS_EXRTO_READ && (buf_state & BM_IS_TMP_BUF)) { + return; + } /* * If we need to protect hint bit updates from torn writes, WAL-log a @@ -5756,9 +5739,8 @@ void MarkBufferDirtyHint(Buffer buffer, bool buffer_std) * The incremental checkpoint is protected by the doublewriter, the * half-write problem does not occur. */ - bool need_write_wal = - (!ENABLE_INCRE_CKPT && XLogHintBitIsNeeded() && (pg_atomic_read_u32(&buf_desc->state) & BM_PERMANENT)); - if (need_write_wal) { + if (unlikely(!ENABLE_INCRE_CKPT && XLogHintBitIsNeeded() && + (pg_atomic_read_u32(&buf_desc->state) & BM_PERMANENT))) { /* * If we're in recovery we cannot dirty a page because of a hint. * We can set the hint, just not dirty the page as a result so the @@ -6197,10 +6179,16 @@ void LockBufferForCleanup(Buffer buffer) */ bool HoldingBufferPinThatDelaysRecovery(void) { - uint32 bufLen = parallel_recovery::GetStartupBufferPinWaitBufLen(); + if (IS_EXRTO_READ) { + return false; + } + SpinLockAcquire(&(g_instance.comm_cxt.predo_cxt.destroy_lock)); int bufids[MAX_RECOVERY_THREAD_NUM + 1]; + errno_t rc = memset_s(bufids, sizeof(bufids), -1, sizeof(bufids)); + securec_check(rc, "\0", "\0"); + uint32 bufLen = parallel_recovery::GetStartupBufferPinWaitBufLen(); parallel_recovery::GetStartupBufferPinWaitBufId(bufids, bufLen); - + SpinLockRelease(&(g_instance.comm_cxt.predo_cxt.destroy_lock)); for (uint32 i = 0; i < bufLen; i++) { /* diff --git a/src/gausskernel/storage/ipc/procarray.cpp b/src/gausskernel/storage/ipc/procarray.cpp index b1bd477c9..ddcbd8f70 100755 --- a/src/gausskernel/storage/ipc/procarray.cpp +++ b/src/gausskernel/storage/ipc/procarray.cpp @@ -543,6 +543,7 @@ void ProcArrayEndTransaction(PGPROC* proc, TransactionId latestXid, bool isCommi proc->snapXmax = InvalidTransactionId; proc->snapCSN = InvalidCommitSeqNo; proc->exrto_read_lsn = 0; + proc->exrto_min = 0; proc->exrto_gen_snap_time = 0; pgxact->csn_min = InvalidCommitSeqNo; pgxact->csn_dr = InvalidCommitSeqNo; @@ -589,6 +590,7 @@ static inline void ProcArrayEndTransactionInternal(PGPROC* proc, PGXACT* pgxact, proc->snapXmax = InvalidTransactionId; proc->snapCSN = InvalidCommitSeqNo; proc->exrto_read_lsn = 0; + proc->exrto_min = 0; proc->exrto_gen_snap_time = 0; pgxact->csn_min = InvalidCommitSeqNo; pgxact->csn_dr = InvalidCommitSeqNo; @@ -819,6 +821,8 @@ void ProcArrayClearTransaction(PGPROC* proc) pgxact->xmin = InvalidTransactionId; proc->snapXmax = InvalidTransactionId; proc->snapCSN = InvalidCommitSeqNo; + proc->exrto_read_lsn = 0; + proc->exrto_gen_snap_time = 0; pgxact->csn_min = InvalidCommitSeqNo; pgxact->csn_dr = InvalidCommitSeqNo; proc->recoveryConflictPending = false; @@ -832,8 +836,7 @@ void ProcArrayClearTransaction(PGPROC* proc) /* Clear the subtransaction-XID cache too */ pgxact->nxids = 0; - proc->exrto_read_lsn = 0; - proc->exrto_gen_snap_time = 0; + proc->exrto_min = 0; /* Free xid cache memory if needed */ ResetProcXidCache(proc, true); } @@ -2111,8 +2114,6 @@ Snapshot GetSnapshotData(Snapshot snapshot, bool force_local_snapshot) bool retry_get = false; uint64 retry_count = 0; const static uint64 WAIT_COUNT = 0x7FFFF; - /* reset xmin before acquiring lwlock, in case blocking redo */ - t_thrd.pgxact->xmin = InvalidTransactionId; RETRY_GET: if (snapshot->takenDuringRecovery && !StreamThreadAmI() && !IS_EXRTO_READ && !u_sess->proc_cxt.clientIsCMAgent) { @@ -2433,15 +2434,40 @@ Snapshot GetSnapshotData(Snapshot snapshot, bool force_local_snapshot) if (snapshot->takenDuringRecovery) { (void)pgstat_report_waitstatus(oldStatus); - } - - if (IsDefaultExtremeRtoMode() && IsExtremeRtoRunning() && pmState == PM_HOT_STANDBY) { - extreme_rto::exrto_read_snapshot(snapshot); + if (IsDefaultExtremeRtoMode() && IS_EXRTO_STANDBY_READ) { + extreme_rto::exrto_read_snapshot(snapshot); + if (t_thrd.proc->exrto_reload_cache) { + t_thrd.proc->exrto_reload_cache = false; + reset_invalidation_cache(); + } + AcceptInvalidationMessages(); + } } return snapshot; } +void exrto_get_snapshot_data(TransactionId &xmin, TransactionId &xmax, CommitSeqNo &snapshot_csn) +{ + LWLockAcquire(ProcArrayLock, LW_SHARED); + + /* xmax is always latest_completed_xid + 1 */ + xmax = t_thrd.xact_cxt.ShmemVariableCache->latestCompletedXid; + + Assert(TransactionIdIsNormal(xmax)); + TransactionIdAdvance(xmax); + /* initialize xmin calculation with xmax */ + xmin = xmax; + if (TransactionIdIsValid(t_thrd.xact_cxt.ShmemVariableCache->standbyXmin)) { + if (TransactionIdPrecedes(t_thrd.xact_cxt.ShmemVariableCache->standbyXmin, xmin)) { + xmin = t_thrd.xact_cxt.ShmemVariableCache->standbyXmin; + } + } + + LWLockRelease(ProcArrayLock); + snapshot_csn = pg_atomic_read_u64(&t_thrd.xact_cxt.ShmemVariableCache->nextCommitSeqNo); +} + /* * ProcArrayInstallImportedXmin -- install imported xmin into MyPgXact->xmin * @@ -3220,10 +3246,10 @@ bool proc_array_cancel_conflicting_proc(TransactionId latest_removed_xid, bool r int pg_proc_no = proc_array->pgprocnos[index]; PGPROC* pg_proc = g_instance.proc_base_all_procs[pg_proc_no]; PGXACT* pg_xact = &g_instance.proc_base_all_xacts[pg_proc_no]; - XLogRecPtr read_lsn = pg_proc->exrto_read_lsn; + XLogRecPtr read_lsn = pg_proc->exrto_min; TransactionId pxmin = pg_xact->xmin; - if (pg_proc->pid == 0 || !TransactionIdIsValid(pxmin) || XLogRecPtrIsInvalid(read_lsn)) { + if (pg_proc->pid == 0 || XLogRecPtrIsInvalid(read_lsn)) { continue; } @@ -3250,6 +3276,10 @@ bool proc_array_cancel_conflicting_proc(TransactionId latest_removed_xid, bool r * Wait a little bit for it to die so that we avoid flooding * an unresponsive backend when system is heavily loaded. */ + ereport(LOG, + (errmsg(EXRTOFORMAT("exrto_gen_snap_time: %ld, current_timestamp: %ld, cancel thread while " + "redo truncate, thread id = %lu\n"), + pg_proc->exrto_gen_snap_time, GetCurrentTimestamp(), pg_proc->pid))); pg_usleep(5000L); } } @@ -4457,11 +4487,11 @@ TransactionId GetGlobal2pcXmin() * Wait for the transaction which modify the tuple to finish. * First release the buffer lock. After waiting, re-acquire the buffer lock. */ -void SyncWaitXidEnd(TransactionId xid, Buffer buffer) +void SyncWaitXidEnd(TransactionId xid, Buffer buffer, const Snapshot snapshot) { if (!BufferIsValid(buffer)) { /* Wait local transaction finish */ - SyncLocalXidWait(xid); + SyncLocalXidWait(xid, snapshot); return; } @@ -4473,7 +4503,7 @@ void SyncWaitXidEnd(TransactionId xid, Buffer buffer) /* Release buffer lock */ LockBuffer(buffer, BUFFER_LOCK_UNLOCK); /* Wait local transaction finish */ - SyncLocalXidWait(xid); + SyncLocalXidWait(xid, snapshot); /* Re-acqure buffer lock, need transform lwlock mode to buffer lock mode */ LockBuffer(buffer, mode == LW_EXCLUSIVE ? BUFFER_LOCK_EXCLUSIVE : BUFFER_LOCK_SHARE); } @@ -4482,7 +4512,7 @@ void SyncWaitXidEnd(TransactionId xid, Buffer buffer) /* * Wait local transaction finish, if transaction wait time exceed transaction_sync_naptime, call gs_clean. */ -void SyncLocalXidWait(TransactionId xid) +void SyncLocalXidWait(TransactionId xid, const Snapshot snapshot) { ReleaseAllGSCRdConcurrentLock(); @@ -4492,7 +4522,7 @@ void SyncLocalXidWait(TransactionId xid) WaitState oldStatus = pgstat_report_waitstatus(STATE_WAIT_UNDEFINED, true); gstrace_entry(GS_TRC_ID_SyncLocalXidWait); - while (!ConditionalXactLockTableWait(xid)) { + while (!ConditionalXactLockTableWait(xid, snapshot)) { /* type of transaction id is same as node id, reuse the second param for waited transaction id */ pgstat_report_waitstatus_xid(STATE_WAIT_XACTSYNC, xid); diff --git a/src/gausskernel/storage/ipc/sinval.cpp b/src/gausskernel/storage/ipc/sinval.cpp index 5954ffc5e..0f82dde70 100644 --- a/src/gausskernel/storage/ipc/sinval.cpp +++ b/src/gausskernel/storage/ipc/sinval.cpp @@ -74,7 +74,7 @@ void GlobalExecuteSharedInvalidMessages(const SharedInvalidationMessage* msgs, i * SendSharedInvalidMessages * Add shared-cache-invalidation message(s) to the global SI message queue. */ -void SendSharedInvalidMessages(const SharedInvalidationMessage* msgs, int n) +void send_shared_invalid_messages(const SharedInvalidationMessage* msgs, int n, XLogRecPtr lsn) { if (ENABLE_DMS && SS_PRIMARY_MODE && !RecoveryInProgress()) { SSSendSharedInvalidMessages(msgs, n); @@ -84,12 +84,17 @@ void SendSharedInvalidMessages(const SharedInvalidationMessage* msgs, int n) if (EnableGlobalSysCache()) { GlobalInvalidSharedInvalidMessages(msgs, n, true); } - SIInsertDataEntries(msgs, n); + SIInsertDataEntries(msgs, n, lsn); if (ENABLE_GPC && g_instance.plan_cache != NULL) { g_instance.plan_cache->InvalMsg(msgs, n); } } +void SendSharedInvalidMessages(const SharedInvalidationMessage* msgs, int n) +{ + send_shared_invalid_messages(msgs, n, 0); +} + static bool SkipRedundantInvalMsg(SharedInvalidationMessage *msg) { if (msg->id != SHAREDINVALRELCACHE_ID || unlikely(u_sess->proc_cxt.MyDatabaseId == InvalidOid) || @@ -171,6 +176,7 @@ void ReceiveSharedInvalidMessages(void (*invalFunction)(SharedInvalidationMessag ereport(DEBUG4, (errmsg("cache state reset"))); inval_cxt->SIMCounter++; resetFunction(); + t_thrd.proc->exrto_reload_cache = true; break; /* nothing more to do */ } diff --git a/src/gausskernel/storage/ipc/sinvaladt.cpp b/src/gausskernel/storage/ipc/sinvaladt.cpp index 630a3155d..490f582c4 100644 --- a/src/gausskernel/storage/ipc/sinvaladt.cpp +++ b/src/gausskernel/storage/ipc/sinvaladt.cpp @@ -179,7 +179,7 @@ typedef struct SISeg { /* * Circular buffer holding shared-inval messages */ - SharedInvalidationMessage buffer[MAXNUMMESSAGES]; + SharedInvalidationMessageEx buffer[MAXNUMMESSAGES]; /* * Per-backend state info. @@ -546,7 +546,7 @@ PGPROC* BackendIdGetProc(int backendID) * SIInsertDataEntries * Add new invalidation message(s) to the buffer. */ -void SIInsertDataEntries(const SharedInvalidationMessage* data, int n) +void SIInsertDataEntries(const SharedInvalidationMessage* data, int n, XLogRecPtr lsn) { SISeg* segP = t_thrd.shemem_ptr_cxt.shmInvalBuffer; @@ -592,7 +592,9 @@ void SIInsertDataEntries(const SharedInvalidationMessage* data, int n) max = segP->maxMsgNum; while (nthistime-- > 0) { - segP->buffer[max % MAXNUMMESSAGES] = *data++; + int index = max % MAXNUMMESSAGES; + segP->buffer[index].msg = *data++; + segP->buffer[index].lsn = lsn; max++; } @@ -735,9 +737,23 @@ int SIGetDataEntries(SharedInvalidationMessage* data, int datasize, bool workses * from the queue. */ n = 0; + + XLogRecPtr read_lsn = InvalidXLogRecPtr; + if (u_sess->utils_cxt.CurrentSnapshot != NULL && + XLogRecPtrIsValid(u_sess->utils_cxt.CurrentSnapshot->read_lsn)) { + read_lsn = u_sess->utils_cxt.CurrentSnapshot->read_lsn; + } else if (XLogRecPtrIsValid(t_thrd.proc->exrto_read_lsn)) { + read_lsn = t_thrd.proc->exrto_read_lsn; + } while (n < datasize && stateP->nextMsgNum < max) { - data[n++] = segP->buffer[stateP->nextMsgNum % MAXNUMMESSAGES]; + int index = stateP->nextMsgNum % MAXNUMMESSAGES; + if (read_lsn != InvalidXLogRecPtr && segP->buffer[index].lsn != InvalidXLogRecPtr) { + if (XLByteLT(read_lsn, segP->buffer[index].lsn)) { + break; + } + } + data[n++] = segP->buffer[index].msg; stateP->nextMsgNum++; } diff --git a/src/gausskernel/storage/lmgr/lmgr.cpp b/src/gausskernel/storage/lmgr/lmgr.cpp index 3154d85bf..00570fb5a 100755 --- a/src/gausskernel/storage/lmgr/lmgr.cpp +++ b/src/gausskernel/storage/lmgr/lmgr.cpp @@ -627,14 +627,18 @@ void XactLockTableWait(TransactionId xid, bool allow_con_update, int waitSec) * As above, but only lock if we can get the lock without blocking. * Returns TRUE if the lock was acquired. */ -bool ConditionalXactLockTableWait(TransactionId xid, bool waitparent, bool bcareNextXid) +bool ConditionalXactLockTableWait(TransactionId xid, const Snapshot snapshot, bool waitparent, bool bcareNextXid) { LOCKTAG tag; CLogXidStatus status = CLOG_XID_STATUS_IN_PROGRESS; + bool takenDuringRecovery = false; + if (snapshot != NULL) { + takenDuringRecovery = snapshot->takenDuringRecovery; + } for (;;) { Assert(!TransactionIdEquals(xid, GetTopTransactionIdIfAny()) || status == CLOG_XID_STATUS_COMMITTED || - status == CLOG_XID_STATUS_ABORTED); + status == CLOG_XID_STATUS_ABORTED || takenDuringRecovery); if (!TransactionIdIsValid(xid)) break; diff --git a/src/gausskernel/storage/lmgr/lwlocknames.txt b/src/gausskernel/storage/lmgr/lwlocknames.txt index 14a519173..06c70bce8 100755 --- a/src/gausskernel/storage/lmgr/lwlocknames.txt +++ b/src/gausskernel/storage/lmgr/lwlocknames.txt @@ -141,3 +141,6 @@ AboCacheLock 131 OndemandXLogMemAllocLock 132 OndemandXLogFileHandleLock 133 ExrtoSnapshotLock 134 +RedoTruncateLock 135 + +ExrtoRecycleResidualUndoLock 137 \ No newline at end of file diff --git a/src/gausskernel/storage/lmgr/proc.cpp b/src/gausskernel/storage/lmgr/proc.cpp index 29ee4bf0b..a4d2050fa 100755 --- a/src/gausskernel/storage/lmgr/proc.cpp +++ b/src/gausskernel/storage/lmgr/proc.cpp @@ -843,6 +843,9 @@ void InitProcess(void) t_thrd.pgxact->xmin = InvalidTransactionId; t_thrd.proc->snapXmax = InvalidTransactionId; t_thrd.proc->snapCSN = InvalidCommitSeqNo; + t_thrd.proc->exrto_read_lsn = 0; + t_thrd.proc->exrto_min = 0; + t_thrd.proc->exrto_gen_snap_time = 0; t_thrd.pgxact->csn_min = InvalidCommitSeqNo; t_thrd.pgxact->csn_dr = InvalidCommitSeqNo; t_thrd.pgxact->prepare_xid = InvalidTransactionId; @@ -950,8 +953,6 @@ void InitProcess(void) t_thrd.proc->snap_refcnt_bitmap = 0; #endif - t_thrd.proc->exrto_read_lsn = 0; - t_thrd.proc->exrto_gen_snap_time = 0; /* Check that group locking fields are in a proper initial state. */ Assert(t_thrd.proc->lockGroupLeader == NULL); Assert(dlist_is_empty(&t_thrd.proc->lockGroupMembers)); @@ -1112,7 +1113,9 @@ void InitAuxiliaryProcess(void) t_thrd.proc->snapXmax = InvalidTransactionId; t_thrd.proc->snapCSN = InvalidCommitSeqNo; t_thrd.proc->exrto_read_lsn = 0; + t_thrd.proc->exrto_min = 0; t_thrd.proc->exrto_gen_snap_time = 0; + t_thrd.proc->exrto_reload_cache = true; t_thrd.pgxact->csn_min = InvalidCommitSeqNo; t_thrd.pgxact->csn_dr = InvalidCommitSeqNo; t_thrd.proc->backendId = InvalidBackendId; diff --git a/src/gausskernel/storage/page/gs_xlogdump.cpp b/src/gausskernel/storage/page/gs_xlogdump.cpp index f76b087ae..852d6be78 100644 --- a/src/gausskernel/storage/page/gs_xlogdump.cpp +++ b/src/gausskernel/storage/page/gs_xlogdump.cpp @@ -107,7 +107,7 @@ static XLogRecPtr GetMaxLSN() return current_recptr; } -static void XLogDumpDisplayRecord(XLogReaderState *record, char *strOutput) +void XLogDumpDisplayRecord(XLogReaderState *record, char *strOutput) { errno_t rc = snprintf_s(strOutput + (int)strlen(strOutput), MAXOUTPUTLEN, MAXOUTPUTLEN - 1, "start_lsn: %X/%X \nend_lsn: %X/%X \nxid: " XID_FMT " \nterm: %u \ntotal length: %u \ndesc: %s - ", @@ -209,7 +209,7 @@ static bool CheckValidRecord(XLogReaderState *xlogreader_state, XLogFilter *filt return found; } -static XLogRecPtr UpdateNextLSN(XLogRecPtr cur_lsn, XLogRecPtr end_lsn, XLogReaderState *xlogreader_state, bool *found) +XLogRecPtr UpdateNextLSN(XLogRecPtr cur_lsn, XLogRecPtr end_lsn, XLogReaderState *xlogreader_state, bool *found) { XLogRecPtr next_record = InvalidXLogRecPtr; for (int tryTimes = 0; tryTimes < FIVE; tryTimes++) { diff --git a/src/gausskernel/storage/replication/basebackup.cpp b/src/gausskernel/storage/replication/basebackup.cpp index e8523cfc6..9432b444f 100755 --- a/src/gausskernel/storage/replication/basebackup.cpp +++ b/src/gausskernel/storage/replication/basebackup.cpp @@ -1780,6 +1780,24 @@ static bool check_data_filename(char *filename, int *segNo) token = strtok_r(filename, "_", &tmptoken); if ('\0' == tmptoken[0]) { + uint dot_count = 0; + int filename_idx = static_cast(strlen(filename) - 1); + // check the last word must be num + if (isdigit(filename[filename_idx]) == false) { + *segNo = 0; + return false; + } + while (filename_idx >= 0) { + if (filename[filename_idx] == '.') { + dot_count++; + } + /* if the char is not num/'.' or dot_count > 1, then break */ + if ((isdigit(filename[filename_idx]) == false && filename[filename_idx] != '.') || dot_count > 1) { + *segNo = 0; + return false; + } + filename_idx--; + } /* MAIN_FORK */ nmatch = sscanf_s(filename, "%u.%d", &relNode, segNo); return (nmatch == 1 || nmatch == 2); diff --git a/src/gausskernel/storage/replication/heartbeat/libpq/fe-connect.cpp b/src/gausskernel/storage/replication/heartbeat/libpq/fe-connect.cpp index e25e12941..2c06069b7 100644 --- a/src/gausskernel/storage/replication/heartbeat/libpq/fe-connect.cpp +++ b/src/gausskernel/storage/replication/heartbeat/libpq/fe-connect.cpp @@ -34,12 +34,14 @@ #define SOCK_ERRNO errno #define SOCK_ERRNO_SET(e) (errno = (e)) +#ifndef FREE_AND_RESET #define FREE_AND_RESET(ptr) do { \ if (NULL != (ptr)) { \ free(ptr); \ (ptr) = NULL; \ } \ } while (0) +#endif namespace PureLibpq { typedef struct PQconninfoOption { diff --git a/src/gausskernel/storage/replication/slot.cpp b/src/gausskernel/storage/replication/slot.cpp index 587929995..7eae7497a 100644 --- a/src/gausskernel/storage/replication/slot.cpp +++ b/src/gausskernel/storage/replication/slot.cpp @@ -692,6 +692,7 @@ void ReplicationSlotRelease(void) t_thrd.pgxact->xmin = InvalidTransactionId; t_thrd.pgxact->vacuumFlags &= ~PROC_IN_LOGICAL_DECODING; t_thrd.proc->exrto_read_lsn = 0; + t_thrd.proc->exrto_min = 0; t_thrd.proc->exrto_gen_snap_time = 0; LWLockRelease(ProcArrayLock); } diff --git a/src/gausskernel/storage/replication/walreceiver.cpp b/src/gausskernel/storage/replication/walreceiver.cpp index ba49b9ae1..b3a7009bf 100755 --- a/src/gausskernel/storage/replication/walreceiver.cpp +++ b/src/gausskernel/storage/replication/walreceiver.cpp @@ -1700,6 +1700,7 @@ static void XLogWalRcvSendHSFeedback(void) xmin = InvalidTransactionId; t_thrd.pgxact->xmin = InvalidTransactionId; t_thrd.proc->exrto_read_lsn = 0; + t_thrd.proc->exrto_min = 0; t_thrd.proc->exrto_gen_snap_time = 0; /* * Always send feedback message. diff --git a/src/gausskernel/storage/replication/walsender.cpp b/src/gausskernel/storage/replication/walsender.cpp index f15b692f6..738d136e4 100755 --- a/src/gausskernel/storage/replication/walsender.cpp +++ b/src/gausskernel/storage/replication/walsender.cpp @@ -2946,6 +2946,7 @@ static void PhysicalReplicationSlotNewXmin(TransactionId feedbackXmin) SpinLockAcquire(&slot->mutex); t_thrd.pgxact->xmin = InvalidTransactionId; t_thrd.proc->exrto_read_lsn = 0; + t_thrd.proc->exrto_min = 0; t_thrd.proc->exrto_gen_snap_time = 0; /* * For physical replication we don't need the the interlock provided diff --git a/src/gausskernel/storage/smgr/md.cpp b/src/gausskernel/storage/smgr/md.cpp index a7e378ee5..082605909 100644 --- a/src/gausskernel/storage/smgr/md.cpp +++ b/src/gausskernel/storage/smgr/md.cpp @@ -25,6 +25,7 @@ #include "miscadmin.h" #include "access/transam.h" #include "access/xlog.h" +#include "access/extreme_rto/standby_read/standby_read_base.h" #include "catalog/catalog.h" #include "portability/instr_time.h" #include "postmaster/bgwriter.h" @@ -1307,7 +1308,7 @@ SMGR_READ_STATUS mdread(SMgrRelation reln, ForkNumber forknum, BlockNumber block } else { check_file_stat(FilePathName(v->mdfd_vfd)); force_backtrace_messages = true; - + extreme_rto_standby_read::dump_error_all_info(reln->smgr_rnode.node, forknum, blocknum); ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read block %u in file \"%s\": read only %d of %d bytes", blocknum, FilePathName(v->mdfd_vfd), nbytes, BLCKSZ))); diff --git a/src/gausskernel/storage/smgr/storage_exrto_file.cpp b/src/gausskernel/storage/smgr/storage_exrto_file.cpp index a3234cee4..46ee9e494 100644 --- a/src/gausskernel/storage/smgr/storage_exrto_file.cpp +++ b/src/gausskernel/storage/smgr/storage_exrto_file.cpp @@ -47,32 +47,12 @@ typedef struct _ExRTOFileState { File file[EXRTO_FORK_NUM]; } ExRTOFileState; -static inline ExRTOFileType exrto_file_type(uint32 space_oid) -{ - if (space_oid == EXRTO_BASE_PAGE_SPACE_OID) { - return BASE_PAGE; - } else if (space_oid == EXRTO_LSN_INFO_SPACE_OID) { - return LSN_INFO_META; - } else { - return BLOCK_INFO_META; - } -} - static inline void set_file_state(ExRTOFileState *state, ForkNumber forknum, uint64 segno, File file) { state->segno[forknum] = segno; state->file[forknum] = file; } -static inline uint64 get_total_block_num(ExRTOFileType type, uint32 high, uint32 low) -{ - if (type == BASE_PAGE || type == LSN_INFO_META) { - return ((uint64)high << UINT64_HALF) | low; - } else { - return (uint64)low; - } -} - static ExRTOFileState *alloc_file_state(void) { MemoryContext current; @@ -100,7 +80,7 @@ static void exrto_get_file_path(const RelFileNode node, ForkNumber forknum, uint if (type == BASE_PAGE || type == LSN_INFO_META) { uint32 batch_id = node.dbNode >> LOW_WORKERID_BITS; uint32 worker_id = node.dbNode & LOW_WORKERID_MASK; - rc = snprintf_s(filename, EXRTO_FILE_PATH_LEN, EXRTO_FILE_PATH_LEN - 1, "%02X%02X%016X", + rc = snprintf_s(filename, EXRTO_FILE_PATH_LEN, EXRTO_FILE_PATH_LEN - 1, "%02x%02x%016lX", batch_id, worker_id, segno); } else { rc = snprintf_s(filename, EXRTO_FILE_PATH_LEN, EXRTO_FILE_PATH_LEN - 1, "%u_%u_%s.%u", @@ -201,16 +181,18 @@ static ExRTOFileState *exrto_open_file(SMgrRelation reln, ForkNumber forknum, Bl return state; } -BlockNumber get_single_file_nblocks(SMgrRelation reln, ForkNumber forknum, const ExRTOFileState*state) +BlockNumber get_single_file_nblocks(SMgrRelation reln, ForkNumber forknum, const ExRTOFileState *state) { Assert(state != NULL); char *filename = FilePathName(state->file[forknum]); off_t len = FileSeek(state->file[forknum], 0L, SEEK_END); if (len < 0) { + char filepath[EXRTO_FILE_PATH_LEN]; + errno_t rc = strcpy_s(filepath, EXRTO_FILE_PATH_LEN, filename); + securec_check(rc, "\0", "\0"); exrto_close(reln, forknum, InvalidBlockNumber); - ereport(ERROR, (errcode_for_file_access(), - errmsg("could not seek to end of file \"%s\": %m", filename))); + ereport(ERROR, (errcode_for_file_access(), errmsg("could not seek to end of file \"%s\": %m", filepath))); } /* note that this calculation will ignore any partial block at EOF */ @@ -235,13 +217,20 @@ void exrto_close(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum) if (state == NULL) { return; } - reln->fileState = NULL; /* prevent dangling pointer after error */ - + /* if not closed already */ if (state->file[forknum] >= 0) { FileClose(state->file[forknum]); + state->file[forknum] = -1; } + for (int forkno = 0; forkno < EXRTO_FORK_NUM; forkno++) { + if (state->file[forkno] != -1) { + return; + } + } + pfree(state); + reln->fileState = NULL; } bool exrto_exists(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum) @@ -260,60 +249,97 @@ bool exrto_exists(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum) return isExist; } -bool exrto_unlink_single_file(const RelFileNodeBackend &rnode, ForkNumber forknum, uint64 segno) +void exrto_unlink_file_with_prefix(char* target_prefix, ExRTOFileType type, uint64 segno) { - struct stat stat_buf; - char segpath[EXRTO_FILE_PATH_LEN]; - - exrto_get_file_path(rnode.node, forknum, segno, segpath); - if (stat(segpath, &stat_buf) < 0) { - if (errno != ENOENT) { - ereport(WARNING, (errcode_for_file_access(), - errmsg("could not stat file \"%s\" before removing: %m", segpath))); - } - return false; + char pathbuf[EXRTO_FILE_PATH_LEN]; + char **filenames; + char **filename; + struct stat statbuf; + /* get file directory */ + char exrto_block_info_dir[EXRTO_FILE_PATH_LEN] = {0}; + int rc = snprintf_s(exrto_block_info_dir, EXRTO_FILE_PATH_LEN, EXRTO_FILE_PATH_LEN - 1, "%s/%s", EXRTO_FILE_DIR, + EXRTO_FILE_SUB_DIR[type]); + securec_check_ss(rc, "", ""); + /* get all files' name from block meta file directory */ + filenames = pgfnames(exrto_block_info_dir); + if (filenames == NULL) { + return; } - if (unlink(segpath) < 0) { - ereport(WARNING, (errcode_for_file_access(), - errmsg("could not remove file \"%s\": %m", segpath))); + + /* use the prefix name to match up files we want to delete */ + size_t prefix_len = strlen(target_prefix); + for (filename = filenames; *filename != NULL; filename++) { + char *fname = *filename; + size_t fname_len = strlen(fname); + /* + * the length of prefix is less than the length of file name and must be the same under the same prefix_len + */ + if (prefix_len >= fname_len || strncmp(target_prefix, fname, prefix_len) != 0) { + continue; + } + if (segno > 0) { + uint32 batch_id, worker_id; + uint64 f_segno; + const int para_num = 3; + if (sscanf_s(fname, "%02X%02X%016lX", &batch_id, &worker_id, &f_segno) != para_num) { + continue; + } + if (f_segno >= segno) { + continue; + } + } + + rc = + snprintf_s(pathbuf, EXRTO_FILE_PATH_LEN, EXRTO_FILE_PATH_LEN - 1, "%s/%s", exrto_block_info_dir, *filename); + securec_check_ss(rc, "", ""); + /* may be can be some error */ + if (lstat(pathbuf, &statbuf) != 0) { + if (errno != ENOENT) { + ereport(WARNING, (errmsg("could not stat file or directory \"%s\" \n", pathbuf))); + } + continue; + } + /* if the file is a directory, don't touch it */ + if (S_ISDIR(statbuf.st_mode)) { + /* skip dir */ + continue; + } + /* delete this file we found */ + if (unlink(pathbuf) != 0) { + if (errno != ENOENT) { + ereport(WARNING, (errmsg("could not remove file or directory \"%s\" ", pathbuf))); + } + } } - return true; + pgfnames_cleanup(filenames); + return; } -void exrto_unlink_file(const RelFileNodeBackend &rnode, ForkNumber forknum, BlockNumber blocknum) +void exrto_unlink(const RelFileNodeBackend &rnode, ForkNumber forknum, bool is_redo, BlockNumber blocknum) { - uint64 segno; + char target_prefix[EXRTO_FILE_PATH_LEN] = {0}; ExRTOFileType type = exrto_file_type(rnode.node.spcNode); + uint64 segno; + errno_t rc; + if (type == BLOCK_INFO_META) { /* unlink all files */ - extreme_rto_standby_read::remove_block_meta_info_files_of_db(rnode.node.dbNode, rnode.node.relNode); + rc = sprintf_s(target_prefix, EXRTO_FILE_PATH_LEN, "%u_%u_", rnode.node.dbNode, rnode.node.relNode); + securec_check_ss(rc, "", ""); + exrto_unlink_file_with_prefix(target_prefix, type); } else if (type == BASE_PAGE || type == LSN_INFO_META) { /* just unlink the files before the file where blocknum is */ segno = get_seg_num(rnode, blocknum); - while (segno != 0) { - segno -= 1; - if (!exrto_unlink_single_file(rnode, forknum, segno)) { - return; - } + if (segno > 0) { + uint32 batch_id = rnode.node.dbNode >> LOW_WORKERID_BITS; + uint32 worker_id = rnode.node.dbNode & LOW_WORKERID_MASK; + rc = sprintf_s(target_prefix, EXRTO_FILE_PATH_LEN, "%02X%02X", batch_id, worker_id); + securec_check_ss(rc, "", ""); + exrto_unlink_file_with_prefix(target_prefix, type, segno); } } } -void exrto_unlink(const RelFileNodeBackend &rnode, ForkNumber forknum, bool is_redo, BlockNumber blocknum) -{ - ExRTOFileType type = exrto_file_type(rnode.node.spcNode); - if (type == BASE_PAGE || type == LSN_INFO_META) { - forknum = MAIN_FORKNUM; - } - if (forknum == InvalidForkNumber) { - for (int fork_num = 0; fork_num < EXRTO_FORK_NUM; fork_num++) { - exrto_unlink_file(rnode, (ForkNumber)fork_num, blocknum); - } - } else { - exrto_unlink_file(rnode, forknum, blocknum); - } -} - /* extend EXTEND_BLOCKS_NUM pages */ void exrto_extend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, char *buffer, bool skip_fsync) { @@ -336,8 +362,11 @@ void exrto_extend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, c state = exrto_open_file(reln, forknum, blocknum, EXTENSION_CREATE); filename = FilePathName(state->file[forknum]); if (stat(filename, &file_stat) < 0) { + char filepath[EXRTO_FILE_PATH_LEN]; + errno_t rc = strcpy_s(filepath, EXRTO_FILE_PATH_LEN, filename); + securec_check(rc, "\0", "\0"); exrto_close(reln, forknum, InvalidBlockNumber); - ereport(ERROR, (errmsg("could not stat file \"%s\": %m.", filename))); + ereport(ERROR, (errmsg("could not stat file \"%s\": %m.", filepath))); } Assert(file_stat.st_size % BLCKSZ == 0); Assert(file_stat.st_size <= EXRTO_FILE_SIZE[type]); @@ -402,14 +431,15 @@ SMGR_READ_STATUS exrto_read(SMgrRelation reln, ForkNumber forknum, BlockNumber b } if (nbytes != BLCKSZ) { char *filename = FilePathName(state->file[forknum]); + char filepath[EXRTO_FILE_PATH_LEN]; + rc = strcpy_s(filepath, EXRTO_FILE_PATH_LEN, filename); + securec_check(rc, "\0", "\0"); exrto_close(reln, forknum, InvalidBlockNumber); if (nbytes < 0) { - ereport(ERROR, - (errmsg("could not read block %u in file \"%s\": %m.", blocknum, filename))); + ereport(ERROR, (errmsg("could not read block %u in file \"%s\": %m.", blocknum, filepath))); } - ereport(ERROR, - (errmsg("could not read block %u in file \"%s\": read only %d of %d bytes.", blocknum, filename, - nbytes, BLCKSZ))); + ereport(ERROR, (errmsg("could not read block %u in file \"%s\": read only %d of %d bytes.", blocknum, filepath, + nbytes, BLCKSZ))); } if (PageIsVerified((Page)buffer, blocknum)) { @@ -441,14 +471,15 @@ void exrto_write(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, co nbytes = FilePWrite(state->file[forknum], buffer, BLCKSZ, seekpos); if (nbytes != BLCKSZ) { char *filename = FilePathName(state->file[forknum]); + char filepath[EXRTO_FILE_PATH_LEN]; + errno_t rc = strcpy_s(filepath, EXRTO_FILE_PATH_LEN, filename); + securec_check(rc, "\0", "\0"); exrto_close(reln, forknum, InvalidBlockNumber); if (nbytes < 0) { - ereport(ERROR, - (errmsg("could not write block %u in file \"%s\": %m.", blocknum, filename))); + ereport(ERROR, (errmsg("could not write block %u in file \"%s\": %m.", blocknum, filepath))); } - ereport(ERROR, - (errmsg("could not write block %u in file \"%s\": wrote only %d of %d bytes.", - blocknum, filename, nbytes, BLCKSZ))); + ereport(ERROR, (errmsg("could not write block %u in file \"%s\": wrote only %d of %d bytes.", blocknum, + filepath, nbytes, BLCKSZ))); } } diff --git a/src/include/access/extreme_rto/dispatcher.h b/src/include/access/extreme_rto/dispatcher.h index ed5e61058..d6b8c8ea6 100644 --- a/src/include/access/extreme_rto/dispatcher.h +++ b/src/include/access/extreme_rto/dispatcher.h @@ -165,7 +165,6 @@ typedef struct { volatile bool recoveryStop; volatile XLogRedoNumStatics xlogStatics[RM_NEXT_ID][MAX_XLOG_INFO_NUM]; RedoTimeCost *startupTimeCost; - ExrtoSnapshotData exrto_snapshot; } LogDispatcher; typedef struct { @@ -183,6 +182,7 @@ extern THR_LOCAL RecordBufferState *g_recordbuffer; const static uint64 OUTPUT_WAIT_COUNT = 0x7FFFFFF; const static uint64 PRINT_ALL_WAIT_COUNT = 0x7FFFFFFFF; +const static uint64 STOP_WORKERS_WAIT_COUNT = 0x13FFFFFFFF; extern RedoItem g_redoEndMark; extern RedoItem g_terminateMark; extern uint32 g_readManagerTriggerFlag; diff --git a/src/include/access/extreme_rto/standby_read/block_info_meta.h b/src/include/access/extreme_rto/standby_read/block_info_meta.h index b1d9eb18a..d4cd7ed73 100644 --- a/src/include/access/extreme_rto/standby_read/block_info_meta.h +++ b/src/include/access/extreme_rto/standby_read/block_info_meta.h @@ -77,8 +77,8 @@ typedef enum { STANDBY_READ_RECLYE_ALL, } StandbyReadRecyleState; -BlockMetaInfo* get_block_meta_info_by_relfilenode( - const BufferTag& buf_tag, BufferAccessStrategy strategy, ReadBufferMode mode, Buffer* buffer); +BlockMetaInfo *get_block_meta_info_by_relfilenode(const BufferTag &buf_tag, BufferAccessStrategy strategy, + ReadBufferMode mode, Buffer *buffer, bool need_share_lock = false); void insert_lsn_to_block_info( StandbyReadMetaInfo* mete_info, const BufferTag& buf_tag, const Page base_page, XLogRecPtr next_lsn); StandbyReadRecyleState recyle_block_info( @@ -98,7 +98,7 @@ static inline bool is_block_meta_info_valid(BlockMetaInfo* meta_info) void remove_one_block_info_file(const RelFileNode rnode); -void remove_block_meta_info_files_of_db(Oid db_oid, Oid rel_oid = InvalidOid); +void remove_block_meta_info_files_of_db(Oid db_oid); } // namespace extreme_rto_standby_read diff --git a/src/include/access/extreme_rto/standby_read/standby_read_base.h b/src/include/access/extreme_rto/standby_read/standby_read_base.h index 714b47506..d61690a93 100644 --- a/src/include/access/extreme_rto/standby_read/standby_read_base.h +++ b/src/include/access/extreme_rto/standby_read/standby_read_base.h @@ -29,6 +29,7 @@ #include "gs_thread.h" #include "postgres.h" #include "storage/buf/bufpage.h" +#include "storage/smgr/smgr.h" #include "postmaster/alarmchecker.h" #define EXRTO_FILE_DIR "standby_read" @@ -67,6 +68,21 @@ typedef struct _StandbyReadMetaInfo { XLogRecPtr recycle_lsn_per_worker; } StandbyReadMetaInfo; +typedef struct WalFilter { + TransactionId by_xid; + bool by_xid_enabled; + bool by_tablepath_enabled; + bool by_block; + bool by_lsn; + RelFileNode by_relfilenode; + BlockNumber block_num; +} WalFilter; + +typedef struct WalPrivate { + const char *data_dir; + TimeLineID tli; +} WalPrivate; + inline void standby_read_meta_page_set_lsn(Page page, XLogRecPtr LSN) { if (XLByteLT(LSN, PageGetLSN(page))) { @@ -75,7 +91,37 @@ inline void standby_read_meta_page_set_lsn(Page page, XLogRecPtr LSN) PageSetLSNInternal(page, LSN); } +inline ExRTOFileType exrto_file_type(uint32 space_oid) +{ + if (space_oid == EXRTO_BASE_PAGE_SPACE_OID) { + return BASE_PAGE; + } else if (space_oid == EXRTO_LSN_INFO_SPACE_OID) { + return LSN_INFO_META; + } else { + return BLOCK_INFO_META; + } +} + +inline uint64 get_total_block_num(ExRTOFileType type, uint32 high, uint32 low) +{ + if (type == BASE_PAGE || type == LSN_INFO_META) { + return ((uint64)high << UINT64_HALF) | low; + } else { + return (uint64)low; + } +} + void exrto_clean_dir(void); void exrto_recycle_old_dir(void); void exrto_standby_read_init(); +void buffer_drop_exrto_standby_read_buffers(StandbyReadMetaInfo *meta_info = NULL); +void exrto_unlink_file_with_prefix(char *target_prefix, ExRTOFileType type, uint64 segno = 0); +extern void XLogDumpDisplayRecord(XLogReaderState *record, char *strOutput); +extern XLogRecPtr UpdateNextLSN(XLogRecPtr cur_lsn, XLogRecPtr end_lsn, XLogReaderState *xlogreader_state, bool *found); +namespace extreme_rto_standby_read { +void dump_error_all_info(const RelFileNode &rnode, ForkNumber forknum, BlockNumber blocknum); +} +#ifdef ENABLE_UT +extern Page get_page_from_buffer(Buffer buf); +#endif #endif \ No newline at end of file diff --git a/src/include/access/extreme_rto/standby_read/standby_read_delay_ddl.h b/src/include/access/extreme_rto/standby_read/standby_read_delay_ddl.h new file mode 100644 index 000000000..ed372174c --- /dev/null +++ b/src/include/access/extreme_rto/standby_read/standby_read_delay_ddl.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * --------------------------------------------------------------------------------------- + * + * standby_read_delay_ddl.h + * + * + * + * IDENTIFICATION + * src/include/access/extreme_rto/standby_read/standby_read_delay_ddl.h + * + * --------------------------------------------------------------------------------------- + */ + +#ifndef STANDBY_READ_DELAY_DDL_H +#define STANDBY_READ_DELAY_DDL_H + +#include "gs_thread.h" +#include "postgres.h" +#include "access/xlogdefs.h" +#include "storage/smgr/relfilenode.h" + +void do_all_old_delay_ddl(); +void delete_by_lsn(XLogRecPtr lsn); +void init_delay_ddl_file(); +void update_delay_ddl_db(Oid db_id, Oid tablespace_id, XLogRecPtr lsn); +void update_delay_ddl_files(ColFileNode* xnodes, int nrels, XLogRecPtr lsn); +void delete_by_table_space(Oid tablespace_id); +#endif \ No newline at end of file diff --git a/src/include/access/multi_redo_api.h b/src/include/access/multi_redo_api.h index 3dbc82dca..0c1bd1f77 100644 --- a/src/include/access/multi_redo_api.h +++ b/src/include/access/multi_redo_api.h @@ -66,8 +66,15 @@ static const uint32 PAGE_REDO_WORKER_READY = 2; static const uint32 PAGE_REDO_WORKER_EXIT = 3; static const uint32 BIG_RECORD_LENGTH = XLOG_BLCKSZ * 16; -#define IS_EXRTO_READ (g_instance.attr.attr_storage.EnableHotStandby && IsExtremeRedo() && IsDefaultExtremeRtoMode()) +#define IS_EXRTO_READ (IsExtremeRedo() && g_instance.attr.attr_storage.EnableHotStandby && IsDefaultExtremeRtoMode()) #define IS_EXRTO_STANDBY_READ (IS_EXRTO_READ && pm_state_is_hot_standby()) +#define IS_EXRTO_RECOVERY_IN_PROGRESS (RecoveryInProgress() && IsExtremeRedo()) + +inline bool is_exrto_standby_read_worker() +{ + return (t_thrd.role == WORKER || t_thrd.role == THREADPOOL_WORKER || t_thrd.role == THREADPOOL_STREAM || + t_thrd.role == STREAM_WORKER); +} static inline int get_real_recovery_parallelism() { diff --git a/src/include/access/rmgrlist.h b/src/include/access/rmgrlist.h index e2e66ffb3..01ae82d15 100644 --- a/src/include/access/rmgrlist.h +++ b/src/include/access/rmgrlist.h @@ -43,7 +43,7 @@ PG_RMGR(RM_MULTIXACT_ID, "MultiXact", multixact_redo, multixact_desc, NULL, NULL PG_RMGR(RM_RELMAP_ID, "RelMap", relmap_redo, relmap_desc, NULL, NULL, NULL, NULL, NULL, relmap_type_name) PG_RMGR(RM_STANDBY_ID, "Standby", standby_redo, standby_desc, StandbyXlogStartup, StandbyXlogCleanup, \ - StandbySafeRestartpoint, NULL, NULL, standby_type_name) + NULL, NULL, NULL, standby_type_name) PG_RMGR(RM_HEAP2_ID, "Heap2", heap2_redo, heap2_desc, NULL, NULL, NULL, NULL, NULL, heap2_type_name) PG_RMGR(RM_HEAP_ID, "Heap", heap_redo, heap_desc, NULL, NULL, NULL, NULL, NULL, heap_type_name) diff --git a/src/include/access/ustore/undo/knl_uundozone.h b/src/include/access/ustore/undo/knl_uundozone.h index 5d87696ae..be6caf8fe 100644 --- a/src/include/access/ustore/undo/knl_uundozone.h +++ b/src/include/access/ustore/undo/knl_uundozone.h @@ -309,7 +309,7 @@ class UndoZone : public BaseObject { } bool CheckNeedSwitch(UndoRecordSize size); UndoRecordState CheckUndoRecordValid(UndoLogOffset offset, bool checkForceRecycle, TransactionId *lastXid); - bool CheckRecycle(UndoRecPtr starturp, UndoRecPtr endurp); + bool CheckRecycle(UndoRecPtr starturp, UndoRecPtr endurp, bool isexrto = false); UndoRecPtr AllocateSpace(uint64 size); void ReleaseSpace(UndoRecPtr starturp, UndoRecPtr endurp, int *forceRecycleSize); @@ -378,5 +378,7 @@ void AllocateZonesBeforXid(); void InitZone(UndoZone *uzone, const int zoneId, UndoPersistence upersistence); void InitUndoSpace(UndoZone *uzone, UndoSpaceType type); bool VerifyUndoZone(UndoZone *uzone); +void exrto_recycle_residual_undo_file(char *FuncName); + } // namespace undo #endif // __KNL_UUNDOZONE_H__ diff --git a/src/include/access/xact.h b/src/include/access/xact.h index 7e502fe2a..09ad0c69a 100644 --- a/src/include/access/xact.h +++ b/src/include/access/xact.h @@ -521,7 +521,9 @@ extern void parseAndRemoveLibrary(char* library, int nlibrary); extern bool IsInLiveSubtransaction(); extern void ExtendCsnlogForSubtrans(TransactionId parent_xid, int nsub_xid, TransactionId* sub_xids); extern CommitSeqNo SetXact2CommitInProgress(TransactionId xid, CommitSeqNo csn); -extern void XactGetRelFiles(XLogReaderState* record, ColFileNode** xnodesPtr, int* nrelsPtr, bool* compress); +extern void XactGetRelFiles(XLogReaderState* record, ColFileNode** xnodesPtr, int* nrelsPtr); +extern bool xact_has_invalid_msg_or_delete_file(XLogReaderState *record); +extern void send_delay_invalid_message(); extern bool XactWillRemoveRelFiles(XLogReaderState *record); extern HTAB* relfilenode_hashtbl_create(); extern CommitSeqNo getLocalNextCSN(); @@ -563,4 +565,6 @@ extern void BeginTxnForAutoCommitOff(); extern void SetTxnInfoForSSLibpqsw(TransactionId xid, CommandId cid); extern void ClearTxnInfoForSSLibpqsw(); extern bool IsTransactionInProgressState(); +extern void unlink_relfiles(_in_ ColFileNode *xnodes, _in_ int nrels); +void xact_redo_log_drop_segs(_in_ ColFileNode *xnodes, _in_ int nrels, XLogRecPtr lsn); #endif /* XACT_H */ diff --git a/src/include/access/xlogproc.h b/src/include/access/xlogproc.h index ed9d926e7..bed5a495f 100755 --- a/src/include/access/xlogproc.h +++ b/src/include/access/xlogproc.h @@ -950,7 +950,7 @@ static inline Buffer AtomicExchangeBuffer(volatile Buffer *ptr, Buffer newval) /* this is an estimated value */ static const uint32 MAX_BUFFER_NUM_PER_WAL_RECORD = XLR_MAX_BLOCK_ID + 1; -static const uint32 LSN_MOVE32 = 10; +static const uint32 LSN_MOVE32 = 32; void HeapXlogCleanOperatorPage( RedoBufferInfo* buffer, void* recorddata, void* blkdata, Size datalen, Size* freespace, bool repairFragmentation); @@ -1277,7 +1277,7 @@ extern void XLogBlockSegDdlDoRealAction(XLogBlockHead* blockhead, void* blockrec extern void GinRedoDataBlock(XLogBlockHead* blockhead, XLogBlockDataParse* blockdatarec, RedoBufferInfo* bufferinfo); extern void GistRedoDataBlock(XLogBlockHead *blockhead, XLogBlockDataParse *blockdatarec, RedoBufferInfo *bufferinfo); extern bool IsCheckPoint(const XLogRecParseState *parseState); - +bool is_backup_end(const XLogRecParseState *parse_state); void redo_atomic_xlog_dispatch(uint8 opCode, RedoBufferInfo *redo_buf, const char *data); void seg_redo_new_page_copy_and_flush(BufferTag *tag, char *data, XLogRecPtr lsn); void redo_target_page(const BufferTag& buf_tag, StandbyReadLsnInfoArray* lsn_info, Buffer base_page_buf); diff --git a/src/include/commands/dbcommands.h b/src/include/commands/dbcommands.h index 1a3fcf021..a394e64e0 100644 --- a/src/include/commands/dbcommands.h +++ b/src/include/commands/dbcommands.h @@ -76,5 +76,6 @@ extern bool IsSetTableSpace(AlterDatabaseStmt* stmt); extern int errdetail_busy_db(int notherbackends, int npreparedxacts); extern void PreCleanAndCheckConns(const char* dbname, bool missing_ok); #endif +extern void do_db_drop(Oid dbId, Oid tbSpcId); #endif /* DBCOMMANDS_H */ diff --git a/src/include/knl/knl_guc/knl_instance_attr_storage.h b/src/include/knl/knl_guc/knl_instance_attr_storage.h index b9c572b81..89ff5a3af 100755 --- a/src/include/knl/knl_guc/knl_instance_attr_storage.h +++ b/src/include/knl/knl_guc/knl_instance_attr_storage.h @@ -213,7 +213,7 @@ typedef struct knl_instance_attr_storage { int64 max_standby_base_page_size; int64 max_standby_lsn_info_size; int base_page_saved_interval; - double standby_force_recyle_ratio; + double standby_force_recycle_ratio; int standby_recycle_interval; int standby_max_query_time; #ifndef ENABLE_MULTIPLE_NODES diff --git a/src/include/knl/knl_instance.h b/src/include/knl/knl_instance.h index 592d4e033..41fdcd7f1 100755 --- a/src/include/knl/knl_instance.h +++ b/src/include/knl/knl_instance.h @@ -113,8 +113,6 @@ enum knl_parallel_redo_state { REDO_DONE, }; -typedef struct ExrtoSnapshotData* ExrtoSnapshot; - /* all process level attribute which expose to user */ typedef struct knl_instance_attr { @@ -728,6 +726,15 @@ typedef struct knl_g_parallel_decode_context { ErrorData *edata; } knl_g_parallel_decode_context; +typedef struct _ExrtoSnapshotData* ExrtoSnapshot; + +typedef struct _StandbyReadDelayDdlState { + uint64 next_index_need_unlink; + uint64 next_index_can_insert; + uint32 delete_stat; + uint32 insert_stat; +} StandbyReadDelayDdlState; + typedef struct knl_g_parallel_redo_context { RedoType redoType; volatile knl_parallel_redo_state state; @@ -751,6 +758,9 @@ typedef struct knl_g_parallel_redo_context { RedoCpuBindControl redoCpuBindcontrl; XLogRecPtr global_recycle_lsn; /* extreme-rto standby read */ HTAB **redoItemHash; /* used in ondemand extreme RTO */ + ExrtoSnapshot exrto_snapshot; + StandbyReadDelayDdlState standby_read_delay_ddl_stat; + uint64 max_clog_pageno; } knl_g_parallel_redo_context; typedef struct knl_g_heartbeat_context { @@ -1404,4 +1414,3 @@ extern void add_numa_alloc_info(void* numaAddr, size_t length); #define DEFAULT_CREATE_GLOBAL_INDEX (u_sess->attr.attr_storage.default_index_kind == DEFAULT_INDEX_KIND_GLOBAL) #endif /* SRC_INCLUDE_KNL_KNL_INSTANCE_H_ */ - diff --git a/src/include/knl/knl_session.h b/src/include/knl/knl_session.h index aaadd977a..c0292d6d2 100644 --- a/src/include/knl/knl_session.h +++ b/src/include/knl/knl_session.h @@ -685,9 +685,6 @@ typedef struct knl_u_utils_context { HTAB* set_user_params_htab; DestReceiver* spi_printtupDR; - - /* backend read lsn for read on standby in extreme rto */ - XLogRecPtr exrto_read_lsn; } knl_u_utils_context; typedef struct knl_u_security_context { diff --git a/src/include/knl/knl_thread.h b/src/include/knl/knl_thread.h index 593420494..e35361a50 100755 --- a/src/include/knl/knl_thread.h +++ b/src/include/knl/knl_thread.h @@ -77,7 +77,6 @@ #include "port/pg_crc32c.h" #include "ddes/dms/ss_common_attr.h" #include "ddes/dms/ss_txnstatus.h" -#include "access/extreme_rto/standby_read/standby_read_base.h" #define MAX_PATH_LEN 1024 extern const int g_reserve_param_num; @@ -1941,19 +1940,31 @@ typedef struct knl_t_conn_context { const char* _float_inf; } knl_t_conn_context; +typedef struct _DelayInvalidMsg { + SharedInvalidationMessage* inval_msgs; + int nmsgs; + uint32 xinfo; + Oid db_id; + Oid ts_id; + XLogRecPtr lsn; + bool relcache_init_file_inval; + bool valid; +} DelayInvalidMsg; + typedef struct { volatile sig_atomic_t shutdown_requested; volatile sig_atomic_t got_SIGHUP; volatile sig_atomic_t sleep_long; volatile sig_atomic_t check_repair; void *redo_worker_ptr; + DelayInvalidMsg invalid_msg; } knl_t_page_redo_context; typedef struct _StandbyReadLsnInfoArray { XLogRecPtr *lsn_array; uint32 lsn_num; XLogRecPtr base_page_lsn; - BasePagePosition base_page_pos; + uint64 base_page_pos; } StandbyReadLsnInfoArray; typedef struct { diff --git a/src/include/storage/buf/bufmgr.h b/src/include/storage/buf/bufmgr.h index 61aa35e54..e3c7b3140 100644 --- a/src/include/storage/buf/bufmgr.h +++ b/src/include/storage/buf/bufmgr.h @@ -35,6 +35,9 @@ #define IsNvmBufferID(id) ((id) >= NvmBufferStartID && (id) < SegmentBufferStartID) #define IsNormalBufferID(id) ((id) >= 0 && (id) < NvmBufferStartID) +#define ExrtoReadStartLSNBktId (-5) +#define ExrtoReadEndLSNBktId (-6) + #define USE_CKPT_THREAD_SYNC (!g_instance.attr.attr_storage.enableIncrementalCheckpoint || \ IsBootstrapProcessingMode() || \ pg_atomic_read_u32(&g_instance.ckpt_cxt_ctl->current_page_writer_count) < 1) @@ -320,7 +323,6 @@ extern void DropRelFileNodeAllBuffersUsingScan(RelFileNode* rnode, int rnode_len extern void DropRelFileNodeOneForkAllBuffersUsingHash(HTAB *relfilenode_hashtbl); extern void DropDatabaseBuffers(Oid dbid); -extern void buffer_drop_exrto_standby_read_buffers(); extern BlockNumber PartitionGetNumberOfBlocksInFork(Relation relation, Partition partition, ForkNumber forkNum, bool estimate = false); diff --git a/src/include/storage/lmgr.h b/src/include/storage/lmgr.h index 8f1c8e236..9de4afff5 100644 --- a/src/include/storage/lmgr.h +++ b/src/include/storage/lmgr.h @@ -66,7 +66,8 @@ extern void UnlockTuple(Relation relation, ItemPointer tid, LOCKMODE lockmode); extern void XactLockTableInsert(TransactionId xid); extern void XactLockTableDelete(TransactionId xid); extern void XactLockTableWait(TransactionId xid, bool allow_con_update = false, int waitSec = 0); -extern bool ConditionalXactLockTableWait(TransactionId xid, bool waitparent = true, bool bCareNextxid = false); +extern bool ConditionalXactLockTableWait(TransactionId xid, const Snapshot snapshot = NULL, bool waitparent = true, + bool bCareNextxid = false); /* Lock a SubXID */ extern void SubXactLockTableInsert(SubTransactionId subxid); diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h index 6b25fa8ce..915f3f791 100755 --- a/src/include/storage/proc.h +++ b/src/include/storage/proc.h @@ -244,6 +244,9 @@ struct PGPROC { /* commit sequence number send down */ CommitSeqNo commitCSN; + XLogRecPtr exrto_read_lsn; /* calculate recycle lsn for read on standby in extreme rto */ + TimestampTz exrto_gen_snap_time; + /* Support for group transaction status update. */ bool clogGroupMember; /* true, if member of clog group */ pg_atomic_uint32 clogGroupNext; /* next clog group member */ @@ -272,8 +275,9 @@ struct PGPROC { uint64 snap_refcnt_bitmap; #endif - XLogRecPtr exrto_read_lsn; /* calculate recycle lsn for read on standby in extreme rto */ - TimestampTz exrto_gen_snap_time; + XLogRecPtr exrto_min; /* calculate recycle lsn for read on standby in extreme rto */ + + bool exrto_reload_cache; LWLock* subxidsLock; struct XidCache subxids; /* cache for subtransaction XIDs */ diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h index 290d88c50..6f0b4aa9d 100755 --- a/src/include/storage/procarray.h +++ b/src/include/storage/procarray.h @@ -50,7 +50,7 @@ extern int GetRoleIdCount(Oid roleoid); extern int IncreaseUserCount(Oid roleoid); extern int DecreaseUserCount(Oid roleoid); -extern void SyncLocalXidWait(TransactionId xid); +extern void SyncLocalXidWait(TransactionId xid, const Snapshot snapshot = NULL); extern Size ProcArrayShmemSize(void); extern void CreateSharedProcArray(void); @@ -84,6 +84,7 @@ Snapshot GetSnapshotData(Snapshot snapshot, bool force_local_snapshot, bool forH #else extern Snapshot GetSnapshotData(Snapshot snapshot, bool force_local_snapshot); #endif +void exrto_get_snapshot_data(TransactionId &xmin, TransactionId &xmax, CommitSeqNo &snapshot_csn); extern Snapshot GetLocalSnapshotData(Snapshot snapshot); @@ -178,7 +179,7 @@ extern void InitProcSubXidCacheContext(); extern void ProcArrayResetXmin(PGPROC* proc); extern uint64 GetCommitCsn(); extern void setCommitCsn(uint64 commit_csn); -extern void SyncWaitXidEnd(TransactionId xid, Buffer buffer); +extern void SyncWaitXidEnd(TransactionId xid, Buffer buffer, const Snapshot snapshot = NULL); extern CommitSeqNo calculate_local_csn_min(); extern void proc_cancel_invalid_gtm_lite_conn(); extern void forward_recent_global_xmin(void); diff --git a/src/include/storage/sinval.h b/src/include/storage/sinval.h index 2e5e404eb..84fbbb4bd 100644 --- a/src/include/storage/sinval.h +++ b/src/include/storage/sinval.h @@ -132,10 +132,16 @@ typedef union SharedInvalidationMessage { SharedInvalFuncMsg fm; } SharedInvalidationMessage; +typedef struct _SharedInvalidationMessageEx { + SharedInvalidationMessage msg; + XLogRecPtr lsn; +} SharedInvalidationMessageEx; + /* Counter of messages processed; don't worry about overflow. */ extern THR_LOCAL volatile sig_atomic_t catchupInterruptPending; extern void SendSharedInvalidMessages(const SharedInvalidationMessage* msgs, int n); +void send_shared_invalid_messages(const SharedInvalidationMessage* msgs, int n, XLogRecPtr lsn); extern void ReceiveSharedInvalidMessages( void (*invalFunction)(SharedInvalidationMessage* msg), void (*resetFunction)(void), bool worksession); @@ -152,7 +158,7 @@ extern void ProcessCatchupInterrupt(void); extern int xactGetCommittedInvalidationMessages(SharedInvalidationMessage** msgs, bool* RelcacheInitFileInval); extern void ProcessCommittedInvalidationMessages( - SharedInvalidationMessage* msgs, int nmsgs, bool RelcacheInitFileInval, Oid dbid, Oid tsid); + SharedInvalidationMessage* msgs, int nmsgs, bool RelcacheInitFileInval, Oid dbid, Oid tsid, XLogRecPtr lsn); extern void LocalExecuteThreadAndSessionInvalidationMessage(SharedInvalidationMessage* msg); extern void LocalExecuteThreadInvalidationMessage(SharedInvalidationMessage* msg); extern void LocalExecuteSessionInvalidationMessage(SharedInvalidationMessage* msg); diff --git a/src/include/storage/sinvaladt.h b/src/include/storage/sinvaladt.h index f77f5bacc..48ede75f5 100644 --- a/src/include/storage/sinvaladt.h +++ b/src/include/storage/sinvaladt.h @@ -34,7 +34,7 @@ extern void CleanupWorkSessionInvalidation(void); extern void SharedInvalBackendInit(bool sendOnly, bool worksession); extern PGPROC* BackendIdGetProc(int backendID); -extern void SIInsertDataEntries(const SharedInvalidationMessage* data, int n); +extern void SIInsertDataEntries(const SharedInvalidationMessage* data, int n, XLogRecPtr lsn = 0); extern int SIGetDataEntries(SharedInvalidationMessage* data, int datasize, bool worksession); extern void SICleanupQueue(bool callerHasWriteLock, int minFree); diff --git a/src/include/storage/smgr/smgr.h b/src/include/storage/smgr/smgr.h index 9acbb1643..288369038 100644 --- a/src/include/storage/smgr/smgr.h +++ b/src/include/storage/smgr/smgr.h @@ -128,7 +128,7 @@ enum SMGR_READ_STATUS { #define EXRTO_BASE_PAGE_SPACE_OID (6) #define EXRTO_LSN_INFO_SPACE_OID (7) #define EXRTO_BLOCK_INFO_SPACE_OID (8) -#define EXRTO_FORK_NUM 3 +#define EXRTO_FORK_NUM MAX_FORKNUM + 1 #define MD_MANAGER (0) #define UNDO_MANAGER (1) diff --git a/src/include/utils/be_module.h b/src/include/utils/be_module.h index d7db7a9c4..a288a1e64 100755 --- a/src/include/utils/be_module.h +++ b/src/include/utils/be_module.h @@ -141,6 +141,8 @@ enum ModuleId { MOD_LOGICAL_DECODE, /* logical decode */ MOD_GPRC, /* global package runtime cache */ MOD_DISASTER_READ, + MOD_STANDBY_READ, + MODE_REPSYNC, /* debug info for func SyncRepWaitForLSN */ MOD_SQLPATCH, MOD_DMS, /* DMS */ diff --git a/src/include/utils/inval.h b/src/include/utils/inval.h index 31b855b09..d49113218 100644 --- a/src/include/utils/inval.h +++ b/src/include/utils/inval.h @@ -77,7 +77,7 @@ extern void CacheInvalidateRelcacheByRelid(Oid relid); extern void CacheInvalidateSmgr(RelFileNodeBackend rnode); -extern void CacheInvalidateRelmap(Oid databaseId); +extern void CacheInvalidateRelmap(Oid databaseId, XLogRecPtr lsn); extern void CacheInvalidateHeapTupleInplace(Relation relation, HeapTuple tuple); @@ -100,5 +100,6 @@ extern void CallSessionSyscacheCallbacks(int cacheid, uint32 hashvalue); extern void InvalidateSessionSystemCaches(void); extern void InvalidateThreadSystemCaches(void); extern void CacheInvalidateRelcacheAll(void); +extern void reset_invalidation_cache(); #endif /* INVAL_H */ diff --git a/src/include/utils/snapmgr.h b/src/include/utils/snapmgr.h index ce1baa4b9..2f6dec245 100644 --- a/src/include/utils/snapmgr.h +++ b/src/include/utils/snapmgr.h @@ -71,6 +71,7 @@ extern void RecheckXidFinish(TransactionId xid, CommitSeqNo csn); extern Snapshot GetTransactionSnapshot(bool force_local_snapshot = false); extern Snapshot GetLatestSnapshot(void); extern Snapshot GetCatalogSnapshot(); +extern Snapshot get_toast_snapshot(); extern void SnapshotSetCommandId(CommandId curcid); extern void PushActiveSnapshot(Snapshot snapshot); @@ -118,4 +119,6 @@ extern void TeardownHistoricSnapshot(bool is_error); extern bool HistoricSnapshotActive(void); extern void SetTransactionSnapshot(Snapshot sourcesnap, VirtualTransactionId *sourcevxid, ThreadId sourcepid); +extern Snapshot get_toast_snapshot(); + #endif /* SNAPMGR_H */ diff --git a/src/include/utils/snapshot.h b/src/include/utils/snapshot.h index 1c38951e6..1ac8a293d 100644 --- a/src/include/utils/snapshot.h +++ b/src/include/utils/snapshot.h @@ -211,7 +211,7 @@ typedef struct SnapshotData { */ TransactionId xmin; /* all XID < xmin are visible to me */ TransactionId xmax; /* all XID >= xmax are invisible to me */ - + XLogRecPtr read_lsn; /* xact lsn when generate snapshot */ /* subxid is in progress and it's the last one modify tuple */ SubTransactionId subxid; @@ -265,9 +265,7 @@ typedef struct SnapshotData { GTM_SnapshotType gtm_snapshot_type; } SnapshotData; -typedef struct ExrtoSnapshotData* ExrtoSnapshot; - -typedef struct ExrtoSnapshotData { +typedef struct _ExrtoSnapshotData { /* * The remaining fields are used only for MVCC snapshots, and are normally * just zeroes in special snapshots. (But xmin and xmax are used @@ -290,6 +288,7 @@ typedef struct ExrtoSnapshotData { XLogRecPtr read_lsn; /* xact lsn when generate snapshot */ TimestampTz gen_snap_time; } ExrtoSnapshotData; +typedef struct _ExrtoSnapshotData *ExrtoSnapshot; /* * Result codes for AM API tuple_{update,delete,lock}, and for visibility. diff --git a/src/test/ha/GNUmakefile b/src/test/ha/GNUmakefile index ee03e462f..69e1ad0a3 100644 --- a/src/test/ha/GNUmakefile +++ b/src/test/ha/GNUmakefile @@ -43,6 +43,9 @@ hacheck_multi_single_mot: all hacheck_multi_single_shared_storage: all export prefix=$(prefix) && sh $(CURDIR)/run_ha_multi_single_shared_storage.sh 2 $(PART) +hacheck_single_standby_read: all + export prefix=$(prefix) && sh $(CURDIR)/run_ha_single_standby_read.sh + ## ## Clean up ## diff --git a/src/test/ha/ha_exrto_standby_read b/src/test/ha/ha_exrto_standby_read new file mode 100644 index 000000000..9aa5316dd --- /dev/null +++ b/src/test/ha/ha_exrto_standby_read @@ -0,0 +1,2 @@ +exrtostandbyread/start_exrto_standby_read +exrtostandbyread/start_exrto_standby_read_multi_data \ No newline at end of file diff --git a/src/test/ha/ha_schedule_single_standby_read b/src/test/ha/ha_schedule_single_standby_read new file mode 100644 index 000000000..d971647cf --- /dev/null +++ b/src/test/ha/ha_schedule_single_standby_read @@ -0,0 +1 @@ +exrtostandbyread/single_standby_read_base \ No newline at end of file diff --git a/src/test/ha/results/exrtostandbyread/.gitignore b/src/test/ha/results/exrtostandbyread/.gitignore new file mode 100644 index 000000000..5e7d2734c --- /dev/null +++ b/src/test/ha/results/exrtostandbyread/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore diff --git a/src/test/ha/run_ha_exrto_standby_read.sh b/src/test/ha/run_ha_exrto_standby_read.sh new file mode 100644 index 000000000..504afa1ce --- /dev/null +++ b/src/test/ha/run_ha_exrto_standby_read.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2010-2018. All rights reserved. +# run all the test case of ha + +#init some variables +loop_num=$1 +if [ -z $1 ]; then + loop_num=1 +fi +count=0 + +source ./standby_env.sh + +total_starttime=`date +"%Y-%m-%d %H:%M:%S"` +total_startvalue=`date -d "$total_starttime" +%s` + +array=("exrtostandbyread") +for element in ${array[@]} +do + mkdir -vp ./results/$element +done + +#init and start the database +if [ "$#" != '2' ]; then + printf "init and start the database\n" + sh deploy_multi_single.sh > ./results/deploy_standby_multi_single.log 2>&1 +fi + +for((i=1;i<=$loop_num;i++)) +do + printf "run the ha_exrto_standby_read %d time\n" $i + printf "%-50s%-10s%-10s\n" "testcase" "result" "time(s)" + for line in `cat ha_exrto_standby_read | grep -v ^#` + do + printf "%-50s" $line + starttime=`date +"%Y-%m-%d %H:%M:%S"` + sh ./testcase/$line.sh > ./results/$line.log 2>&1 + count=`expr $count + 1` + endtime=`date +"%Y-%m-%d %H:%M:%S"` + starttime1=`date -d "$starttime" +%s` + endtime1=`date -d "$endtime" +%s` + interval=`expr $endtime1 - $starttime1` + if [ $( grep "$failed_keyword" ./results/$line.log | grep -v "the database system is shutting down" | wc -l ) -eq 0 ]; then + printf "%-10s%-10s\n" ".... ok" $interval + else + printf "%-10s%-10s\n" ".... FAILED" $interval + # exit 0 + fi + done +done + +#stop the database +printf "stop the database\n" +python $scripts_dir/pgxc_multi.py -o > ./results/stop_database_multi.log 2>&1 + +total_endtime=`date +"%Y-%m-%d %H:%M:%S"` +total_endvalue=`date -d "$total_endtime" +%s` +printf "all %d tests passed.\n" $count +printf "total time: %ss\n" $(($total_endvalue - $total_startvalue)) diff --git a/src/test/ha/run_ha_single_standby_read.sh b/src/test/ha/run_ha_single_standby_read.sh new file mode 100644 index 000000000..8c9035d4f --- /dev/null +++ b/src/test/ha/run_ha_single_standby_read.sh @@ -0,0 +1,50 @@ +#!/bin/sh +# run the test case of standby-read + +#init some variables +count=0 +loop_num=$1 +if [ -z $1 ]; then + loop_num=1 +fi + +source ./standby_env.sh + +total_starttime=`date +"%Y-%m-%d %H:%M:%S"` +total_startvalue=`date -d "$total_starttime" +%s` + +#init and start the database +printf "init and start the database\n" +sh deploy_multi_single.sh > ./results/deploy_standby_multi_single.log 2>&1 + +for((i=1;i<=$loop_num;i++)) +do + printf "run the ha_schedule %d time\n" $i + printf "%-50s%-10s%-10s\n" "testcase" "result" "time(s)" + for line in `cat ha_schedule_single_standby_read$2 | grep -v ^#` + do + printf "%-50s" $line + starttime=`date +"%Y-%m-%d %H:%M:%S"` + starttime1=`date -d "$starttime" +%s` + sh ./testcase/$line.sh > ./results/$line.log 2>&1 + endtime=`date +"%Y-%m-%d %H:%M:%S"` + endtime1=`date -d "$endtime" +%s` + interval=`expr $endtime1 - $starttime1` + if [ $( grep "$failed_keyword" ./results/$line.log | grep -v "the database system is shutting down" | wc -l ) -eq 0 ]; then + printf "%-10s%-10s\n" ".... ok" $interval + count=`expr $count + 1` + else + printf "%-10s%-10s\n" ".... FAILED" $interval + echo "hacheck mistakes info in " ./results/$line.log + fi + done +done + +#stop the database +printf "stop the database\n" +python $scripts_dir/pgxc_psd_single.py -o > ./results/stop_database_single.log 2>&1 + +total_endtime=`date +"%Y-%m-%d %H:%M:%S"` +total_endvalue=`date -d "$total_endtime" +%s` +printf "all %d tests passed.\n" $count +printf "total time: %ss\n" $(($total_endvalue - $total_startvalue)) diff --git a/src/test/ha/standby_env.sh b/src/test/ha/standby_env.sh index c9e37b85e..05743bfe7 100644 --- a/src/test/ha/standby_env.sh +++ b/src/test/ha/standby_env.sh @@ -1,7 +1,7 @@ #!/bin/sh #some enviroment vars -export g_base_port=8888 +export g_base_port=25632 export prefix=${GAUSSHOME} export g_pooler_base_port=`expr $g_base_port \+ 410` export g_base_standby_port=`expr $g_base_port \+ 400` diff --git a/src/test/ha/testcase/exrtostandbyread/single_standby_read_base.sh b/src/test/ha/testcase/exrtostandbyread/single_standby_read_base.sh new file mode 100644 index 000000000..56fcd6bde --- /dev/null +++ b/src/test/ha/testcase/exrtostandbyread/single_standby_read_base.sh @@ -0,0 +1,144 @@ +#!/bin/sh + +source ./util.sh + +function check_select_result() +{ + if [ $(echo $result | grep "${1}" | wc -l) -eq 1 ]; then + echo "remote read successful" + else + echo "remote read failed $failed_keyword with [$result]" + exit 1 + fi +} + +function test_base_sql_func() +{ + gsql -d test_standby_read_base -p $dn1_primary_port -c "DROP TABLE if exists test1; CREATE TABLE test1(contentId VARCHAR(128) NOT NULL, commentId VARCHAR(128) NOT NULL, appId VARCHAR(128) NOT NULL, PRIMARY KEY (contentId, commentId)) with(parallel_workers=8,storage_type=aSTORE);" + gsql -d test_standby_read_base -p $dn1_primary_port -c "DROP TABLE if exists test2; CREATE TABLE test2(contentId VARCHAR(128) NOT NULL, commentId VARCHAR(128) NOT NULL, appId VARCHAR(128) NOT NULL, PRIMARY KEY (contentId, commentId)) with(storage_type=aSTORE,fillfactor=80) partition by hash(contentId);" + gsql -d test_standby_read_base -p $dn1_primary_port -c "DROP TABLE if exists test3; CREATE TABLE test3(contentId VARCHAR(128) NOT NULL, commentId VARCHAR(128) NOT NULL, appId VARCHAR(128) NOT NULL, PRIMARY KEY (contentId, commentId)) with(storage_type=uSTORE,fillfactor=40) partition by list(contentId) (partition p1 values ('1') ,partition p2 values ('2') ,partition p3 values ('3') ,partition p4 values (default));" + + gsql -d test_standby_read_base -p $dn1_primary_port -c "insert into test1 select generate_series(1,20), generate_series(1,20), generate_series(1,20);" + gsql -d test_standby_read_base -p $dn1_primary_port -c "insert into test2 select generate_series(1,300), generate_series(1,300), generate_series(1,300);" + gsql -d test_standby_read_base -p $dn1_primary_port -c "insert into test3 select generate_series(1,20), generate_series(1,20), generate_series(1,20);" + + gsql -d test_standby_read_base -p $dn1_primary_port -c "delete test1 where contentId = 8;" + gsql -d test_standby_read_base -p $dn1_primary_port -c "delete test2 where contentId = 8;" + gsql -d test_standby_read_base -p $dn1_primary_port -c "delete test3 where contentId = 8;" + + gsql -d test_standby_read_base -p $dn1_primary_port -c "update test1 set appId = 2 where contentId = 1;" + gsql -d test_standby_read_base -p $dn1_primary_port -c "update test2 set appId = 2 where contentId = 1;" + gsql -d test_standby_read_base -p $dn1_primary_port -c "update test2 set appId = 2 where contentId = 1;" + + sleep 5 + + echo "execute light-proxy plan" + result=`gsql -d test_standby_read_base -p $dn1_standby_port -c "select * from test1 where appId = 10;"` + check_select_result "10 | 10 | 10 (1 row)" + + echo "execute remote-query plan" + result=`gsql -d test_standby_read_base -p $dn1_standby_port -c "select count(*) from test1, test2 where test1.appId = test2.appId;"` + check_select_result "21 (1 row)" + + echo "execute pbe fqs plan" + result=`gsql -d test_standby_read_base -p $dn1_standby_port -c 'prepare a as select * from test3 where appId = $1;execute a (10);'` + check_select_result "10 | 10 | 10 (1 row)" + + + echo "execute pbe remote-query plan" + result=`gsql -d test_standby_read_base -p $dn1_standby_port -c 'prepare b as select count(*) from test3, test1 where test3.appId = $1;execute b (10);'` + check_select_result "19 (1 row)" + + echo "execute cursor" + result=`gsql -d test_standby_read_base -p $dn1_standby_port -c "START TRANSACTION;CURSOR cursor1 FOR SELECT * FROM test1 order by 1;FETCH 1 IN cursor1;CLOSE cursor1;END;"` + check_select_result "1 | 1 | 2 (1 row) CLOSE CURSOR COMMIT" + + echo "execute transaction" + gsql -d test_standby_read_base -p $dn1_primary_port -c 'begin; update test2 set test2.appId = test2.contentId + 10 where test2.contentId = 10; select pg_sleep(2); commit;' & + result=`gsql -d test_standby_read_base -p $dn1_standby_port -c "select appId from test2 where test2.contentId = 10;"` + check_select_result "10 (1 row)" + sleep 5 + result=`gsql -d test_standby_read_base -p $dn1_standby_port -c "select appId from test2 where test2.contentId = 10;"` + check_select_result "20 (1 row)" + gsql -d test_standby_read_base -p $dn1_primary_port -c 'update test2 set test2.appId = 10 where test2.contentId = 10; select pg_sleep(3);' +} + +function test_standby_read_base_func() +{ + set_default + kill_cluster + + echo "set guc" + gs_guc set -Z datanode -D $primary_data_dir -c "recovery_parse_workers = 1" + gs_guc set -Z datanode -D $primary_data_dir -c "recovery_redo_workers = 1" + gs_guc set -Z datanode -D $primary_data_dir -c "hot_standby = on" + + gs_guc set -Z datanode -D $standby_data_dir -c "recovery_parse_workers = 1" + gs_guc set -Z datanode -D $standby_data_dir -c "recovery_redo_workers = 1" + gs_guc set -Z datanode -D $standby_data_dir -c "hot_standby = on" + + start_cluster + echo "start cluster success" + sleep 2 + + echo "prepare data" + gsql -d $db -p $dn1_primary_port -c "create database test_standby_read_base with encoding='UTF8' template=template0;" + + echo "test serial redo" + + test_base_sql_func + + sleep 2 + + echo "test paraller redo" + + gs_guc set -Z datanode -D $primary_data_dir -c "recovery_parse_workers = 1" + gs_guc set -Z datanode -D $standby_data_dir -c "recovery_parse_workers = 1" + + gs_guc set -Z datanode -D $primary_data_dir -c " recovery_max_workers = 4" + gs_guc set -Z datanode -D $standby_data_dir -c " recovery_max_workers = 4" + + kill_cluster + start_cluster + + test_base_sql_func + + sleep 2 + + echo "test exrto redo" + + gs_guc set -Z datanode -D $primary_data_dir -c "recovery_parse_workers = 2" + gs_guc set -Z datanode -D $standby_data_dir -c "recovery_parse_workers = 2" + + gs_guc set -Z datanode -D $primary_data_dir -c " recovery_redo_workers = 4" + gs_guc set -Z datanode -D $standby_data_dir -c " recovery_redo_workers = 4" + + kill_cluster + start_cluster + + test_base_sql_func +} + +function tear_down() { + sleep 1 + gsql -d $db -p $cn1_port -c "DROP DATABASE if exists test_standby_read_base;" + + stop_streaming_cluster + + echo "reset guc" + + gs_guc set -Z datanode -D $primary_data_dir -c "recovery_parse_workers=1" + gs_guc set -Z datanode -D $standby_data_dir -c "recovery_parse_workers=1" + + gs_guc set -Z datanode -D $primary_data_dir -c " recovery_redo_workers = 1" + gs_guc set -Z datanode -D $standby_data_dir -c " recovery_redo_workers = 1" + + gs_guc set -Z datanode -D $primary_data_dir -c " recovery_max_workers = 1" + gs_guc set -Z datanode -D $standby_data_dir -c " recovery_max_workers = 1" + + gs_guc set -Z datanode -D $primary_data_dir -c "hot_standby = on" + gs_guc set -Z datanode -D $standby_data_dir -c "hot_standby = on" +} + +test_standby_read_base_func +tear_down \ No newline at end of file diff --git a/src/test/ha/testcase/exrtostandbyread/start_exrto_standby_read.sh b/src/test/ha/testcase/exrtostandbyread/start_exrto_standby_read.sh new file mode 100644 index 000000000..e9d97222c --- /dev/null +++ b/src/test/ha/testcase/exrtostandbyread/start_exrto_standby_read.sh @@ -0,0 +1,81 @@ +source ./util.sh + +function test_1() +{ + set_default + kill_cluster + printf "set extreme_rto_standby_read para\n" + gs_guc set -Z datanode -D $primary_data_dir -c "recovery_parse_workers = 3" + gs_guc set -Z datanode -D $primary_data_dir -c "recovery_redo_workers = 3" + gs_guc set -Z datanode -D $primary_data_dir -c "hot_standby = on" + + gs_guc set -Z datanode -D $standby_data_dir -c "recovery_parse_workers = 3" + gs_guc set -Z datanode -D $standby_data_dir -c "recovery_redo_workers = 3" + gs_guc set -Z datanode -D $standby_data_dir -c "hot_standby = on" + start_cluster + echo "start cluster success" + sleep 5 + + echo "insert data on primary" + gsql -d postgres -p ${dn1_primary_port} -m -c "drop table aaa;" + gsql -d postgres -p ${dn1_primary_port} -m -c "create table aaa(number int);" + gsql -d postgres -p ${dn1_primary_port} -m -c "insert into aaa(number) values(100);" + + echo "primary guc check" + res=`gsql -d postgres -p ${dn1_primary_port} -m -c "show recovery_parse_workers \x" | grep recovery_parse_workers | awk '{print $NF}'` + if [ "$res" != 3 ]; then + echo "extreme_rto_standby_read is off 1, $failed_keyword" + exit 1 + fi + + res=`gsql -d postgres -p ${dn1_primary_port} -m -c "show recovery_redo_workers \x" | grep recovery_redo_workers | awk '{print $NF}'` + if [ "$res" != 3 ]; then + echo "extreme_rto_standby_read is off 2, $failed_keyword" + exit 1 + fi + + res=`gsql -d postgres -p ${dn1_primary_port} -m -c "show hot_standby \x" | grep hot_standby | awk '{print $NF}'` + if [ "$res" != "on" ]; then + echo "extreme_rto_standby_read is off 3, $failed_keyword" + exit 1 + fi + + echo "standby guc check" + res=`gsql -d postgres -p ${dn1_standby_port} -m -c "show recovery_parse_workers \x" | grep recovery_parse_workers | awk '{print $NF}'` + if [ "$res" != 3 ]; then + echo "extreme_rto_standby_read is off 4, $failed_keyword" + exit 1 + fi + + res=`gsql -d postgres -p ${dn1_standby_port} -m -c "show recovery_redo_workers \x" | grep recovery_redo_workers | awk '{print $NF}'` + if [ "$res" != 3 ]; then + echo "extreme_rto_standby_read is off 5, $failed_keyword" + exit 1 + fi + + res=`gsql -d postgres -p ${dn1_standby_port} -m -c "show hot_standby \x" | grep hot_standby | awk '{print $NF}'` + if [ "$res" != "on" ]; then + echo "extreme_rto_standby_read is off 6, $failed_keyword" + exit 1 + fi + + echo "query data on standby" + res=`gsql -d postgres -p ${dn1_standby_port} -m -c "select * from aaa;" -x | grep number | awk '{print $NF}'` + if [ "$res" != 100 ]; then + echo "extreme_rto_standby_read is off 7, $failed_keyword" + exit 1 + else + echo "extreme_rto_standby_read is running" + fi +} + +function tear_down() { + stop_streaming_cluster + gs_guc set -Z datanode -D $primary_data_dir -c "recovery_parse_workers = 1" + gs_guc set -Z datanode -D $primary_data_dir -c "recovery_redo_workers = 1" + + gs_guc set -Z datanode -D $standby_data_dir -c "recovery_parse_workers = 1" + gs_guc set -Z datanode -D $standby_data_dir -c "recovery_redo_workers = 1" +} +test_1 +tear_down diff --git a/src/test/ha/testcase/exrtostandbyread/start_exrto_standby_read_multi_data.sh b/src/test/ha/testcase/exrtostandbyread/start_exrto_standby_read_multi_data.sh new file mode 100644 index 000000000..776774994 --- /dev/null +++ b/src/test/ha/testcase/exrtostandbyread/start_exrto_standby_read_multi_data.sh @@ -0,0 +1,91 @@ +source ./util.sh + +function test_1() +{ + set_default + kill_cluster + printf "set extreme_rto_standby_read para\n" + gs_guc set -Z datanode -D $primary_data_dir -c "recovery_parse_workers = 3" + gs_guc set -Z datanode -D $primary_data_dir -c "recovery_redo_workers = 3" + gs_guc set -Z datanode -D $primary_data_dir -c "hot_standby = on" + + gs_guc set -Z datanode -D $standby_data_dir -c "recovery_parse_workers = 3" + gs_guc set -Z datanode -D $standby_data_dir -c "recovery_redo_workers = 3" + gs_guc set -Z datanode -D $standby_data_dir -c "hot_standby = on" + start_cluster + echo "start cluster success" + sleep 5 + + echo "insert data on primary" + gsql -d postgres -p ${dn1_primary_port} -m -c "drop table aaa;" + gsql -d postgres -p ${dn1_primary_port} -m -c "create table aaa(number int);" + for((i=1;i<=100;i++)) + do + gsql -d postgres -p ${dn1_primary_port} -m -c "insert into aaa(number) values($i);" + done + + echo "primary guc check" + res=`gsql -d postgres -p ${dn1_primary_port} -m -c "show recovery_parse_workers \x" | grep recovery_parse_workers | awk '{print $NF}'` + if [ "$res" != 3 ]; then + echo "extreme_rto_standby_read is off 1, $failed_keyword" + exit 1 + fi + + res=`gsql -d postgres -p ${dn1_primary_port} -m -c "show recovery_redo_workers \x" | grep recovery_redo_workers | awk '{print $NF}'` + if [ "$res" != 3 ]; then + echo "extreme_rto_standby_read is off 2, $failed_keyword" + exit 1 + fi + + res=`gsql -d postgres -p ${dn1_primary_port} -m -c "show hot_standby \x" | grep hot_standby | awk '{print $NF}'` + if [ "$res" != "on" ]; then + echo "extreme_rto_standby_read is off 3, $failed_keyword" + exit 1 + fi + + echo "standby guc check" + res=`gsql -d postgres -p ${dn1_standby_port} -m -c "show recovery_parse_workers \x" | grep recovery_parse_workers | awk '{print $NF}'` + if [ "$res" != 3 ]; then + echo "extreme_rto_standby_read is off 4, $failed_keyword" + exit 1 + fi + + res=`gsql -d postgres -p ${dn1_standby_port} -m -c "show recovery_redo_workers \x" | grep recovery_redo_workers | awk '{print $NF}'` + if [ "$res" != 3 ]; then + echo "extreme_rto_standby_read is off 5, $failed_keyword" + exit 1 + fi + + res=`gsql -d postgres -p ${dn1_standby_port} -m -c "show hot_standby \x" | grep hot_standby | awk '{print $NF}'` + if [ "$res" != "on" ]; then + echo "extreme_rto_standby_read is off 6, $failed_keyword" + exit 1 + fi + + echo "query data on standby" + gsql -d postgres -p ${dn1_standby_port} -m -c "select * from aaa;" -x | grep number | awk '{print $NF}' > ./results/exrtostandbyread/start_exrto_standby_read_multi_data.txt + if [ "$?" -ne "0" ]; then + echo "extreme_rto_standby_read is off 7, $failed_keyword" + exit 1 + fi + number=0 + for line in $(cat ./results/exrtostandbyread/start_exrto_standby_read_multi_data.txt) + do + let number++ + if [ $line != $number ]; then + echo "extreme_rto_standby_read query data is wrong, $line, $number, $failed_keyword" + exit 1 + fi + done +} + +function tear_down() { + stop_streaming_cluster + gs_guc set -Z datanode -D $primary_data_dir -c "recovery_parse_workers = 1" + gs_guc set -Z datanode -D $primary_data_dir -c "recovery_redo_workers = 1" + + gs_guc set -Z datanode -D $standby_data_dir -c "recovery_parse_workers = 1" + gs_guc set -Z datanode -D $standby_data_dir -c "recovery_redo_workers = 1" +} +test_1 +tear_down diff --git a/src/test/regress/CMakeLists.txt b/src/test/regress/CMakeLists.txt index 10ddb4731..1e959290c 100755 --- a/src/test/regress/CMakeLists.txt +++ b/src/test/regress/CMakeLists.txt @@ -61,6 +61,7 @@ add_func_target_withargs_fastcheck(hacheck_decode hacheck_decode) add_func_target_withargs_fastcheck(hacheck_multi_single hacheck_multi_single) add_func_target_withargs_fastcheck(hacheck_multi_single_mot hacheck_multi_single_mot) add_func_target_withargs_fastcheck(hacheck_single_paxos hacheck_single_paxos) +add_func_target_withargs_fastcheck(hacheck_single_standby_read hacheck_single_standby_read) add_func_target_withargs_fastcheck(decodecheck_single decodecheck_single) add_func_target_withargs_fastcheck(upgradecheck_single upgradecheck_single) add_func_target_withargs_fastcheck(hacheck_ss_all hacheck_ss_all) diff --git a/src/test/regress/parallel_schedule0 b/src/test/regress/parallel_schedule0 index 3a3de2b2d..6935231d7 100644 --- a/src/test/regress/parallel_schedule0 +++ b/src/test/regress/parallel_schedule0 @@ -12,7 +12,7 @@ test: btree_deduplication # This test case is used to monitor GUC parameter information. # If the GUC parameter is changed, please modify the Code/src/bin/gs_guc/cluster_guc.conf and Code/src/test/regress/output/recovery_2pc_tools.source files. #test: recovery_2pc_tools recovery_2pc_tools02 -test: recovery_2pc_tools +#test: recovery_2pc_tools test: sqlbypass_partition test: sqlpatch_base diff --git a/src/test/regress/parallel_schedule0A b/src/test/regress/parallel_schedule0A index 64d6d228f..76da7e9e0 100644 --- a/src/test/regress/parallel_schedule0A +++ b/src/test/regress/parallel_schedule0A @@ -9,7 +9,7 @@ # This test case is used to monitor GUC parameter information. # If the GUC parameter is changed, please modify the Code/src/bin/gs_guc/cluster_guc.conf and Code/src/test/regress/output/recovery_2pc_tools.source files. #test: recovery_2pc_tools recovery_2pc_tools02 -test: recovery_2pc_tools +#test: recovery_2pc_tools test: sqlbypass_partition test: sqlpatch_base diff --git a/src/test/regress/single_check.sh b/src/test/regress/single_check.sh index 03d19a1a4..203eeda94 100755 --- a/src/test/regress/single_check.sh +++ b/src/test/regress/single_check.sh @@ -265,6 +265,8 @@ function real_hacheck() sh ./run_paxos_single.sh ;; hacheck_ss_all) sh ./run_ha_single_ss.sh ;; + hacheck_single_standby_read) + sh ./run_ha_single_standby_read.sh ;; *) echo "module $module is not valid" ;; esac @@ -520,7 +522,7 @@ case $DO_CMD in --wlmcheck_single|wlmcheck_single) args_val="-d 6 -c 3 -p $p -r ${runtest}" real_wmlcheck parallel_schedule${part}.wlm make_wlmcheck_postgresql.conf "${args_val}" ;; - --hacheck_single_all|hacheck_single_all|--hacheck_single|hacheck_single|--hacheck_multi_single|hacheck_multi_single|--hacheck_multi_single_mot|hacheck_multi_single_mot|--hacheck_decode|hacheck_decode|--hacheck_single_paxos|hacheck_single_paxos|--hacheck_ss_all|hacheck_ss_all) + --hacheck_single_all|hacheck_single_all|--hacheck_single|hacheck_single|--hacheck_multi_single|hacheck_multi_single|--hacheck_multi_single_mot|hacheck_multi_single_mot|--hacheck_decode|hacheck_decode|--hacheck_single_paxos|hacheck_single_paxos|--hacheck_ss_all|hacheck_ss_all|--hacheck_single_standby_read|hacheck_single_standby_read) args_val=$(echo $DO_CMD | sed 's\--\\g') real_hacheck "${args_val}";; --fastcheck_ledger_single|fastcheck_ledger_single) From a0b7bb843d571e4e786236e89268d05c5028db71 Mon Sep 17 00:00:00 2001 From: vastdata-xyzr Date: Tue, 12 Sep 2023 17:01:51 +0800 Subject: [PATCH 253/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dnbtutils.cpp=E4=B8=AD?= =?UTF-8?q?=E9=94=99=E8=AF=AF=E7=9A=84ereport=E5=86=99=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/storage/access/nbtree/nbtutils.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/gausskernel/storage/access/nbtree/nbtutils.cpp b/src/gausskernel/storage/access/nbtree/nbtutils.cpp index c8df4a0fa..93d3ffba9 100644 --- a/src/gausskernel/storage/access/nbtree/nbtutils.cpp +++ b/src/gausskernel/storage/access/nbtree/nbtutils.cpp @@ -2068,14 +2068,14 @@ void btree_check_third_page(Relation rel, Relation heap, bool need_heaptid_space errmsg("index row size %lu exceeds maximum %lu for index \"%s\"", (unsigned long)itemsz, (unsigned long)BTREE_MAX_ITEM_SIZE(page), - RelationGetRelationName(rel), + RelationGetRelationName(rel)), errdetail("Index row references tuple (%u,%hu) in relation \"%s\".", ItemPointerGetBlockNumber(&tuple->t_tid), ItemPointerGetOffsetNumber(&tuple->t_tid), heap ? RelationGetRelationName(heap) : "unknown"), errhint("Values larger than 1/3 of a buffer page cannot be indexed.\n" "Consider a function index of an MD5 hash of the value, " - "or use full text indexing.")))); + "or use full text indexing."))); } bool btree_allequalimage(Relation rel, bool debugmessage) From 462af9fb897429273202b6ee772d485f7e0593b2 Mon Sep 17 00:00:00 2001 From: bowenliu Date: Wed, 13 Sep 2023 16:37:50 +0800 Subject: [PATCH 254/304] =?UTF-8?q?=E9=80=82=E9=85=8D=E6=96=B0MES?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/ddes/adapter/ss_dms_callback.cpp | 6 ++++++ src/gausskernel/ddes/adapter/ss_init.cpp | 9 ++++----- src/gausskernel/ddes/ddes_commit_id | 6 +++--- src/include/ddes/dms/dms_api.h | 13 +++++++------ src/include/knl/knl_thread.h | 1 + 5 files changed, 21 insertions(+), 14 deletions(-) diff --git a/src/gausskernel/ddes/adapter/ss_dms_callback.cpp b/src/gausskernel/ddes/adapter/ss_dms_callback.cpp index 2a7a04b92..66dbc94fb 100644 --- a/src/gausskernel/ddes/adapter/ss_dms_callback.cpp +++ b/src/gausskernel/ddes/adapter/ss_dms_callback.cpp @@ -497,6 +497,11 @@ static void ReleaseResource() FlushErrorState(); } +static unsigned int CBIncAndGetSrsn(uint32 sessid) +{ + return ++t_thrd.dms_cxt.srsn; +} + static unsigned int CBPageHashCode(const char pageid[DMS_PAGEID_SIZE]) { BufferTag *tag = (BufferTag *)pageid; @@ -2066,6 +2071,7 @@ void DmsInitCallback(dms_callback_t *callback) callback->reform_set_dms_role = CBReformSetDmsRole; callback->opengauss_ondemand_redo_buffer = CBOndemandRedoPageForStandby; + callback->inc_and_get_srsn = CBIncAndGetSrsn; callback->get_page_hash_val = CBPageHashCode; callback->read_local_page4transfer = CBEnterLocalPage; callback->leave_local_page = CBLeaveLocalPage; diff --git a/src/gausskernel/ddes/adapter/ss_init.cpp b/src/gausskernel/ddes/adapter/ss_init.cpp index 4c84acae2..f4ca57d78 100644 --- a/src/gausskernel/ddes/adapter/ss_init.cpp +++ b/src/gausskernel/ddes/adapter/ss_init.cpp @@ -12,11 +12,11 @@ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. * See the Mulan PSL v2 for more details. * --------------------------------------------------------------------------------------- - * + * * ss_init.cpp * initialize for DMS shared storage. - * - * + * + * * IDENTIFICATION * src/gausskernel/ddes/adapter/ss_init.cpp * @@ -302,7 +302,7 @@ static void setScrlConfig(dms_profile_t *profile) // server bind (void)setBindCoreConfig(dms_attr->scrlock_server_bind_core_config, &profile->scrlock_server_bind_core_start, &profile->scrlock_server_bind_core_end); - + // worker bind if (setBindCoreConfig(dms_attr->scrlock_worker_bind_core_config, &profile->scrlock_worker_bind_core_start, &profile->scrlock_worker_bind_core_end)) { @@ -375,7 +375,6 @@ static void setDMSProfile(dms_profile_t* profile) SetOckLogPath(dms_attr, profile->ock_log_path); profile->inst_map = 0; profile->enable_reform = (unsigned char)dms_attr->enable_reform; - profile->load_balance_mode = 1; /* primary-standby */ profile->parallel_thread_num = dms_attr->parallel_thread_num; profile->max_wait_time = DMS_MSG_MAX_WAIT_TIME; diff --git a/src/gausskernel/ddes/ddes_commit_id b/src/gausskernel/ddes/ddes_commit_id index 8dafe7b09..6f5522f0e 100644 --- a/src/gausskernel/ddes/ddes_commit_id +++ b/src/gausskernel/ddes/ddes_commit_id @@ -1,3 +1,3 @@ -dms_commit_id=d7095d9b169a766461b00aa4bc0e19b4fc8ea657 -dss_commit_id=0a759717486f91cc91b19ca6b0cf0d2e48f08fcb -cbb_commit_id=a8ac49e215cfe26098416b44721fe76eaca2bc1d \ No newline at end of file +dms_commit_id=1f1cacdc3285a8b43b1e73f0f439f79dda7630d0 +dss_commit_id=a4d52e326a5291ca1ef8d2d22993a561f13f45a8 +cbb_commit_id=a7f55f6371988695578304bfbd3450f6f31c6630 \ No newline at end of file diff --git a/src/include/ddes/dms/dms_api.h b/src/include/ddes/dms/dms_api.h index 03bd86b2d..17d443753 100644 --- a/src/include/ddes/dms/dms_api.h +++ b/src/include/ddes/dms/dms_api.h @@ -221,6 +221,7 @@ typedef struct st_dms_context { unsigned char is_try; unsigned char type; unsigned short len; + unsigned long long ctx_ruid; /* this ruid indicates one message ack is pending recv */ union { char resid[DMS_RESID_SIZE]; dms_drid_t lock_id; @@ -607,6 +608,7 @@ typedef int(*dms_tx_area_load)(void *db_handle, unsigned char inst_id); typedef int(*dms_tx_rollback_finish)(void *db_handle, unsigned char inst_id); typedef unsigned char(*dms_recovery_in_progress)(void *db_handle); typedef unsigned int(*dms_get_page_hash_val)(const char pageid[DMS_PAGEID_SIZE]); +typedef unsigned int(*dms_inc_and_get_srsn)(unsigned int sess_id); typedef unsigned long long(*dms_get_page_lsn)(const dms_buf_ctrl_t *buf_ctrl); typedef int(*dms_set_buf_load_status)(dms_buf_ctrl_t *buf_ctrl, dms_buf_load_status_t dms_buf_load_status); typedef int(*dms_remove_buf_load_status)(dms_buf_ctrl_t *buf_ctrl, dms_buf_load_status_t dms_buf_load_status); @@ -691,7 +693,6 @@ typedef void (*dms_check_if_build_complete)(void *db_handle, unsigned int *build typedef void (*dms_check_if_restore_recover)(void *db_handle, unsigned int *rst_recover); typedef int (*dms_db_is_primary)(void *db_handle); typedef void (*dms_set_switchover_result)(void *db_handle, int result); -typedef void (*dms_set_db_role)(void *db_handle, unsigned char is_primary); typedef int (*dms_mount_to_recovery)(void *db_handle, unsigned int *has_offline); typedef int(*dms_get_open_status)(void *db_handle); typedef void (*dms_reform_set_dms_role)(void *db_handle, unsigned int reformer_id); @@ -779,6 +780,7 @@ typedef struct st_dms_callback { dms_reform_start_notify reform_start_notify; dms_reform_set_dms_role reform_set_dms_role; + dms_inc_and_get_srsn inc_and_get_srsn; dms_get_page_hash_val get_page_hash_val; dms_get_page_lsn get_page_lsn; dms_set_buf_load_status set_buf_load_status; @@ -862,7 +864,6 @@ typedef struct st_dms_callback { dms_switchover_promote_opengauss switchover_promote_opengauss; dms_failover_promote_opengauss failover_promote_opengauss; dms_set_switchover_result set_switchover_result; - dms_set_db_role set_db_role; dms_mount_to_recovery mount_to_recovery; dms_reform_done_notify reform_done_notify; @@ -887,6 +888,7 @@ typedef struct st_dms_callback { } dms_callback_t; typedef struct st_dms_instance_net_addr { + unsigned int inst_id; char ip[DMS_MAX_IP_LEN]; unsigned short port; unsigned char reserved[2]; @@ -912,9 +914,8 @@ typedef struct st_dms_profile { // Indicates whether to connected to other instances during DMS initialization. unsigned int conn_created_during_init : 1; unsigned int resource_catalog_centralized : 1; // 1: centralized, 0: distributed - unsigned int load_balance_mode : 1; // 1: primary&standby unsigned int time_stat_enabled : 1; - unsigned int reserved : 28; + unsigned int reserved : 29; unsigned int elapsed_switch; unsigned char rdma_rpc_use_busypoll; // busy poll need to occupy the cpu core unsigned char rdma_rpc_is_bind_core; @@ -925,7 +926,7 @@ typedef struct st_dms_profile { // ock scrlock configs unsigned char enable_scrlock; unsigned int primary_inst_id; - unsigned char enable_ssl; + unsigned char enable_ssl; unsigned int scrlock_log_level; unsigned char enable_scrlock_worker_bind_core; unsigned int scrlock_worker_cnt; @@ -967,7 +968,7 @@ typedef enum st_dms_protocol_version { #define DMS_LOCAL_MINOR_VER_WEIGHT 1000 #define DMS_LOCAL_MAJOR_VERSION 0 #define DMS_LOCAL_MINOR_VERSION 0 -#define DMS_LOCAL_VERSION 92 +#define DMS_LOCAL_VERSION 94 #ifdef __cplusplus } diff --git a/src/include/knl/knl_thread.h b/src/include/knl/knl_thread.h index e35361a50..bba36f6dd 100755 --- a/src/include/knl/knl_thread.h +++ b/src/include/knl/knl_thread.h @@ -3393,6 +3393,7 @@ typedef struct knl_t_dms_context { bool flush_copy_get_page_failed; //used in flush copy HTAB* SSTxnStatusHash; LRUQueue* SSTxnStatusLRU; + uint32 srsn; /* session rsn used for DMS page request ordering */ } knl_t_dms_context; typedef struct knl_t_ondemand_xlog_copy_context { From 86f3837969b6d6ce6c9c3e92379d4b390f3a1c71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BD=98=E5=85=B4=E5=BD=AC?= Date: Tue, 19 Sep 2023 10:45:40 +0800 Subject: [PATCH 255/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=9F=A5=E8=AF=A2?= =?UTF-8?q?=E9=9B=86=E7=BE=A4reform=E4=BF=A1=E6=81=AF=E8=BE=93=E5=87=BA?= =?UTF-8?q?=E6=9C=89=E8=AF=AF=E7=9A=84BUG?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/utils/adt/pgstatfuncs.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/backend/utils/adt/pgstatfuncs.cpp b/src/common/backend/utils/adt/pgstatfuncs.cpp index c8946edbf..a2288dcb7 100644 --- a/src/common/backend/utils/adt/pgstatfuncs.cpp +++ b/src/common/backend/utils/adt/pgstatfuncs.cpp @@ -14983,7 +14983,7 @@ Datum query_node_reform_info(PG_FUNCTION_ARGS) reform_iterate_t *iterate = (reform_iterate_t *)funcctx->user_fctx; ss_reform_info_t reform_info = iterate->reform_info; for (uint64 i = iterate->iterate_idx; i < DMS_MAX_INSTANCE; i++) { - if (!((reform_info.old_bitmap | reform_info.new_bitmap) & (1 << i))) { + if (!((reform_info.old_bitmap | reform_info.new_bitmap) & (((uint64)1) << i))) { continue; } From 2174500076c91bade76b724424ae02d2cf5a3623 Mon Sep 17 00:00:00 2001 From: hwhbj Date: Tue, 19 Sep 2023 15:09:22 +0800 Subject: [PATCH 256/304] =?UTF-8?q?=20=E4=BF=AE=E5=A4=8Dpagehack,=20pg=5Fx?= =?UTF-8?q?logdump=E5=AE=89=E5=85=A8=E7=BC=96=E8=AF=91=E9=80=89=E9=A1=B9?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- contrib/pagehack/Makefile | 6 +++--- contrib/pg_xlogdump/Makefile | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/contrib/pagehack/Makefile b/contrib/pagehack/Makefile index 4c944b767..ccd3ee1cb 100644 --- a/contrib/pagehack/Makefile +++ b/contrib/pagehack/Makefile @@ -19,11 +19,11 @@ subdir = contrib/pagehack top_builddir = ../.. include $(top_builddir)/src/Makefile.global override CPPFLAGS += -I${top_builddir}/src/lib/page_compression -override LDFLAGS += -L${top_builddir}/src/lib/page_compression -override CFLAGS += -lpagecompression -lzstd -llz4 +override LDFLAGS += -L${top_builddir}/src/lib/page_compression -Wl,-z,relro,-z,now +override CFLAGS += -lpagecompression -lzstd -llz4 -fstack-protector-all ifeq ($(enable_debug), yes) PG_CPPFLAGS += -DDEBUG endif include $(top_srcdir)/contrib/contrib-global.mk -endif +endif \ No newline at end of file diff --git a/contrib/pg_xlogdump/Makefile b/contrib/pg_xlogdump/Makefile index 61a89991d..67c7b5278 100644 --- a/contrib/pg_xlogdump/Makefile +++ b/contrib/pg_xlogdump/Makefile @@ -21,7 +21,9 @@ include $(top_builddir)/src/Makefile.global include $(top_srcdir)/contrib/contrib-global.mk -override CPPFLAGS := -DFRONTEND $(CPPFLAGS) +override CPPFLAGS := -DFRONTEND $(CPPFLAGS) -fstack-protector-all -Wl,-z,relro,-z,now +override LDFLAGS += -Wl,-z,relro,-z,now +override CFLAGS += -fstack-protector-all xlogreader.cpp: % : $(top_srcdir)/src/gausskernel/storage/access/transam/% rm -f $@ && $(LN_S) $< . From ee4908a8288eae2034c7bb018338a5c949160d68 Mon Sep 17 00:00:00 2001 From: maxiang Date: Tue, 19 Sep 2023 17:08:28 +0800 Subject: [PATCH 257/304] add rsa2048 and sm2 back for client encryption --- .../encrypt/security_aead_aes_hmac_enc_key.cpp | 8 -------- .../keymgr/encrypt/security_sm2_enc_key.cpp | 14 ++++++++++++++ .../keymgr/localkms/security_localkms.cpp | 16 +++++----------- .../encrypt/security_aead_aes_hamc_enc_key.h | 8 +++++++- 4 files changed, 26 insertions(+), 20 deletions(-) diff --git a/src/gausskernel/security/keymgr/encrypt/security_aead_aes_hmac_enc_key.cpp b/src/gausskernel/security/keymgr/encrypt/security_aead_aes_hmac_enc_key.cpp index db55850b0..8528473e0 100644 --- a/src/gausskernel/security/keymgr/encrypt/security_aead_aes_hmac_enc_key.cpp +++ b/src/gausskernel/security/keymgr/encrypt/security_aead_aes_hmac_enc_key.cpp @@ -47,14 +47,6 @@ const unsigned char *AeadAesHamcEncKey::g_iv_key_salt_format = const int RAND_COUNT = 100; -void HmacCtxGroup::free_hmac_ctx(HMAC_CTX** ctx_tmp) const -{ - if (*ctx_tmp != NULL) { - HMAC_CTX_free(*ctx_tmp); - *ctx_tmp = NULL; - } -} - /* Derives all the required keys from the given root key */ AeadAesHamcEncKey::AeadAesHamcEncKey(unsigned char *root_key, size_t root_key_size) { diff --git a/src/gausskernel/security/keymgr/encrypt/security_sm2_enc_key.cpp b/src/gausskernel/security/keymgr/encrypt/security_sm2_enc_key.cpp index 350e9898d..a1c1f27a3 100644 --- a/src/gausskernel/security/keymgr/encrypt/security_sm2_enc_key.cpp +++ b/src/gausskernel/security/keymgr/encrypt/security_sm2_enc_key.cpp @@ -161,6 +161,13 @@ CmkemErrCode encrypt_with_sm2_pubkey(CmkemUStr *plain, CmkemUStr *pub_key, Cmkem return CMKEM_EVP_ERR; } + ret = EVP_PKEY_set_alias_type(public_evp_key, EVP_PKEY_SM2); + if (ret != 1) { + cmkem_errmsg("EVP_PKEY_set_alias_type to EVP_PKEY_SM2 failed!"); + EVP_PKEY_free(public_evp_key); + return CMKEM_EVP_ERR; + } + /* do cipher. */ ctx = EVP_PKEY_CTX_new(public_evp_key, NULL); EVP_PKEY_free(public_evp_key); @@ -244,6 +251,13 @@ CmkemErrCode decrypt_with_sm2_privkey(CmkemUStr *cipher, CmkemUStr *priv_key, Cm return CMKEM_EVP_ERR; } + ret = EVP_PKEY_set_alias_type(private_evp_key, EVP_PKEY_SM2); + if (ret != 1) { + cmkem_errmsg("EVP_PKEY_set_alias_type to EVP_PKEY_SM2 failed!"); + EVP_PKEY_free(private_evp_key); + return CMKEM_EVP_ERR; + } + /* do cipher. */ ctx = EVP_PKEY_CTX_new(private_evp_key, NULL); EVP_PKEY_free(private_evp_key); diff --git a/src/gausskernel/security/keymgr/localkms/security_localkms.cpp b/src/gausskernel/security/keymgr/localkms/security_localkms.cpp index 69f8ae440..079031f69 100644 --- a/src/gausskernel/security/keymgr/localkms/security_localkms.cpp +++ b/src/gausskernel/security/keymgr/localkms/security_localkms.cpp @@ -45,7 +45,7 @@ const int MAX_KEY_PATH_LEN = 64; const int MIN_KEY_PATH_LEN = 1; -static const char *g_support_algo[] = {"RSA_3072", "SM2", NULL}; +static const char *g_support_algo[] = {"RSA_2048", "RSA_3072", "SM2", NULL}; LocalKmsMgr *localkms_new(KmErr *err) { @@ -427,7 +427,7 @@ void kms_mk_create(KeyMgr *kmgr, KeyInfo info) switch (get_algo_by_str(info.algo)) { case AT_RSA_2048: - km_err_msg(kms->kmgr.err, "rsa_2048 is not safe now, please use rsa_3072 instead."); + ret = create_and_write_rsa_key_pair(info.id, RSA2048_KEN_LEN); return; case AT_RSA_3072: ret = create_and_write_rsa_key_pair(info.id, RSA3072_KEN_LEN); @@ -493,11 +493,6 @@ char *kms_mk_select(KeyMgr *kmgr, KeyInfo info) return NULL; } - if (strcasecmp(info.algo, "RSA_2048") == 0) { - km_err_msg(kms->kmgr.err, "rsa_2048 is not safe now, please use rsa_3072 instead."); - return NULL; - } - ret = check_cmk_algo_validity(info.algo); if (ret != CMKEM_SUCCEED) { km_err_msg(kms->kmgr.err, "%s", get_cmkem_errmsg(ret)); @@ -518,9 +513,8 @@ KmUnStr kms_mk_encrypt(KeyMgr *kmgr, KeyInfo info, KmUnStr plain) switch (get_algo_by_str(info.algo)) { case AT_RSA_2048: - km_err_msg(kms->kmgr.err, "the algorithm of master key is rsa_2048, but rsa_2048 is not safe now, " - "please create new master key with rsa_3072."); - return cipher; + ret = encrypt_cek_with_rsa(&_plain, info.id, &_cipher); + break; case AT_RSA_3072: ret = encrypt_cek_with_rsa(&_plain, info.id, &_cipher); break; @@ -551,7 +545,7 @@ KmUnStr kms_mk_decrypt(KeyMgr *kmgr, KeyInfo info, KmUnStr cipher) CmkemUStr *_plain = NULL; switch (get_algo_by_str(info.algo)) { - case AT_RSA_2048: /* only decrypt the existing data encrypted by old version of opengauss */ + case AT_RSA_2048: case AT_RSA_3072: ret = decrypt_cek_with_rsa(&_cipher, info.id, &_plain); break; diff --git a/src/include/keymgr/encrypt/security_aead_aes_hamc_enc_key.h b/src/include/keymgr/encrypt/security_aead_aes_hamc_enc_key.h index b66c0307d..d556ab459 100644 --- a/src/include/keymgr/encrypt/security_aead_aes_hamc_enc_key.h +++ b/src/include/keymgr/encrypt/security_aead_aes_hamc_enc_key.h @@ -49,7 +49,13 @@ class HmacCtxGroup { HMAC_CTX* ctx_worker; HMAC_CTX* ctx_template; private: - void free_hmac_ctx(HMAC_CTX** ctx_tmp) const; + void free_hmac_ctx(HMAC_CTX** ctx_tmp) + { + if (*ctx_tmp != NULL) { + HMAC_CTX_free(*ctx_tmp); + *ctx_tmp = NULL; + } + } }; /* From ae57aacdfb2385f5db803373a72bf8691cfefc3d Mon Sep 17 00:00:00 2001 From: liuzhanfeng2 Date: Tue, 19 Sep 2023 16:47:42 +0800 Subject: [PATCH 258/304] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E8=B5=84=E6=BA=90?= =?UTF-8?q?=E6=B1=A0=E5=8C=96=E5=9C=BA=E6=99=AF=E4=B8=8Bgs=5Fprobackup?= =?UTF-8?q?=E5=B7=A5=E5=85=B7=E4=B8=ADDSS=E8=AF=BB=E5=86=99=E6=AD=A5?= =?UTF-8?q?=E9=95=BF,=E4=BC=98=E5=8C=96=E6=80=A7=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/pg_probackup/data.cpp | 184 +++++++++++++++++++++++++++++++--- src/bin/pg_probackup/file.cpp | 46 +++++++-- src/bin/pg_probackup/file.h | 5 +- 3 files changed, 208 insertions(+), 27 deletions(-) diff --git a/src/bin/pg_probackup/data.cpp b/src/bin/pg_probackup/data.cpp index 52e71a19c..bd1e6a970 100644 --- a/src/bin/pg_probackup/data.cpp +++ b/src/bin/pg_probackup/data.cpp @@ -31,6 +31,12 @@ #include "zstd.h" #include "storage/file/fio_device.h" +typedef struct PreReadBuf +{ + int num; + char *data; +} PreReadBuf; + /* Union to ease operations on relation pages */ typedef struct DataPage { @@ -466,7 +472,7 @@ prepare_page(ConnectionArgs *conn_arg, Page page, bool strict, uint32 checksum_version, const char *from_fullpath, - PageState *page_st, PageCompression *pageCompression, int &read_len) + PageState *page_st, PageCompression *pageCompression, int &read_len, PreReadBuf *preReadBuf) { int try_again = PAGE_READ_ATTEMPTS; bool page_is_valid = false; @@ -475,7 +481,10 @@ prepare_page(ConnectionArgs *conn_arg, /* check for interrupt */ if (is_interrupt) + { + pg_free(preReadBuf->data); elog(ERROR, "Interrupted during page reading"); + } /* * Read the page and verify its header and checksum. @@ -486,7 +495,54 @@ prepare_page(ConnectionArgs *conn_arg, while (!page_is_valid && try_again--) { /* read the block */ - read_len = fio_pread(in, page, blknum * BLCKSZ, pageCompression); + int offset = blknum * BLCKSZ; + int fileStartOff = offset - (offset % DSS_BLCKSZ); + if (IsDssMode() && file->size - fileStartOff >= DSS_BLCKSZ) + { + int preReadOff = offset % DSS_BLCKSZ; + if (offset / DSS_BLCKSZ == preReadBuf->num) + { + rc = memcpy_s(page, BLCKSZ, preReadBuf->data + preReadOff, BLCKSZ); + securec_check(rc, "\0", "\0"); + read_len = BLCKSZ; + } + else + { + read_len = fio_pread(in, preReadBuf->data, fileStartOff, pageCompression, DSS_BLCKSZ); + preReadBuf->num = offset / DSS_BLCKSZ; + if (read_len == 0) + { + elog(VERBOSE, "Cannot read block %u of \"%s\": " + "block truncated", offset / DSS_BLCKSZ, from_fullpath); + } + else if (read_len < 0) + { + pg_free(preReadBuf->data); + elog(ERROR, "Cannot read block %u of \"%s\": %s", + offset / DSS_BLCKSZ, from_fullpath, strerror(errno)); + } + else if (read_len != DSS_BLCKSZ) + { + if (read_len > (int)MIN_COMPRESS_ERROR_RT) + { + pg_free(preReadBuf->data); + elog(ERROR, "Cannot read block %u of \"%s\" code: %lu : %s", offset / DSS_BLCKSZ, from_fullpath, read_len, + strerror(errno)); + } + elog(WARNING, + "Cannot read block %u of \"%s\": " + "read %i of %d, try again", + offset / DSS_BLCKSZ, from_fullpath, read_len, DSS_BLCKSZ); + } + rc = memcpy_s(page, BLCKSZ, preReadBuf->data + preReadOff, BLCKSZ); + securec_check(rc, "\0", "\0"); + read_len = BLCKSZ; + } + } + else + { + read_len = fio_pread(in, page, blknum * BLCKSZ, pageCompression, BLCKSZ); + } /* The block could have been truncated. It is fine. */ if (read_len == 0) @@ -496,10 +552,14 @@ prepare_page(ConnectionArgs *conn_arg, return PageIsTruncated; } else if (read_len < 0) + { + pg_free(preReadBuf->data); elog(ERROR, "Cannot read block %u of \"%s\": %s", blknum, from_fullpath, strerror(errno)); + } else if (read_len != BLCKSZ) { if (read_len > (int)MIN_COMPRESS_ERROR_RT) { + pg_free(preReadBuf->data); elog(ERROR, "Cannot read block %u of \"%s\" code: %lu : %s", blknum, from_fullpath, read_len, strerror(errno)); } @@ -573,11 +633,19 @@ prepare_page(ConnectionArgs *conn_arg, elevel = WARNING; if (errormsg) + { + if (elevel == ERROR) + pg_free(preReadBuf->data); elog(elevel, "Corruption detected in file \"%s\", block %u: %s", from_fullpath, blknum, errormsg); + } else + { + if (elevel == ERROR) + pg_free(preReadBuf->data); elog(elevel, "Corruption detected in file \"%s\", block %u", from_fullpath, blknum); + } pg_free(errormsg); return PageIsCorrupted; @@ -1007,6 +1075,13 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers elog(ERROR, "Cannot seek block %u of \"%s\": %s", blknum, to_fullpath, strerror(errno)); + char *preWriteBuf = (char*)malloc(DSS_BLCKSZ); + if (preWriteBuf == NULL) + elog(ERROR, "malloc preWriteBuf failed, size : %d", DSS_BLCKSZ); + int preWriteOff = 0; + int targetSize = file->write_size; + int *p_preWriteOff = &preWriteOff; + int *p_targetSize = &targetSize; for (;;) { off_t write_pos; @@ -1022,7 +1097,10 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers /* check for interrupt */ if (interrupted || thread_interrupted) + { + pg_free(preWriteBuf); elog(ERROR, "Interrupted during data file restore"); + } /* newer backups have headers in separate storage */ if (headers) @@ -1099,14 +1177,23 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers /* To correctly truncate file, we must first flush STDIO buffers */ if (fio_fflush(out) != 0) + { + pg_free(preWriteBuf); elog(ERROR, "Cannot flush file \"%s\": %s", to_fullpath, strerror(errno)); + } /* Set position to the start of file */ if (fio_fseek(out, 0) < 0) + { + pg_free(preWriteBuf); elog(ERROR, "Cannot seek to the start of file \"%s\": %s", to_fullpath, strerror(errno)); + } if (fio_ftruncate(out, blknum * BLCKSZ) != 0) + { + pg_free(preWriteBuf); elog(ERROR, "Cannot truncate file \"%s\": %s", to_fullpath, strerror(errno)); + } break; } @@ -1119,7 +1206,10 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers break; if (compressed_size > BLCKSZ) + { + pg_free(preWriteBuf); elog(ERROR, "Size of a blknum %i exceed BLCKSZ: %i", blknum, compressed_size); + } /* Incremental restore in LSN mode */ if (map && lsn_map && datapagemap_is_set(lsn_map, blknum)) @@ -1147,8 +1237,11 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers * go to the next page. */ if (!headers && fseek(in, read_len, SEEK_CUR) != 0) + { + pg_free(preWriteBuf); elog(ERROR, "Cannot seek block %u of \"%s\": %s", blknum, from_fullpath, strerror(errno)); + } continue; } @@ -1156,8 +1249,11 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers cur_pos_in != headers[n_hdr].pos) { if (fseek(in, headers[n_hdr].pos, SEEK_SET) != 0) + { + pg_free(preWriteBuf); elog(ERROR, "Cannot seek to offset %u of \"%s\": %s", headers[n_hdr].pos, from_fullpath, strerror(errno)); + } cur_pos_in = headers[n_hdr].pos; } @@ -1169,8 +1265,11 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers len = fread(page.data, 1, read_len, in); if (len != read_len) + { + pg_free(preWriteBuf); elog(ERROR, "Cannot read block %u file \"%s\": %s", blknum, from_fullpath, strerror(errno)); + } cur_pos_in += read_len; @@ -1197,8 +1296,11 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers if (cur_pos_out != write_pos) { if (fio_fseek(out, write_pos) < 0) + { + pg_free(preWriteBuf); elog(ERROR, "Cannot seek block %u of \"%s\": %s", blknum, to_fullpath, strerror(errno)); + } cur_pos_out = write_pos; } @@ -1206,20 +1308,62 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers /* If page is compressed and restore is in remote mode, send compressed * page to the remote side. */ - if (is_compressed) + if (IsDssMode() && targetSize >= DSS_BLCKSZ) { - ssize_t rc; - rc = fio_fwrite_compressed(out, page.data, compressed_size, file->compress_alg); + if (is_compressed) + { + ssize_t rc; + rc = fio_fwrite_compressed(out, page.data, compressed_size, file->compress_alg, to_fullpath, preWriteBuf, p_preWriteOff, p_targetSize); - if (!fio_is_remote_file(out) && rc != BLCKSZ) - elog(ERROR, "Cannot write block %u of \"%s\": %s, size: %u", - blknum, to_fullpath, strerror(errno), compressed_size); + if (!fio_is_remote_file(out) && rc != BLCKSZ) + { + pg_free(preWriteBuf); + elog(ERROR, "Cannot write block %u of preWriteBuf: %s, size: %u", + blknum, strerror(errno), compressed_size); + } + } + else + { + int ret = memcpy_s(preWriteBuf + preWriteOff, DSS_BLCKSZ, page.data, BLCKSZ); + securec_check(ret, "\0", "\0"); + preWriteOff += BLCKSZ; + + if (preWriteOff == DSS_BLCKSZ) + { + if (fio_fwrite(out, preWriteBuf, DSS_BLCKSZ) != DSS_BLCKSZ) + { + pg_free(preWriteBuf); + elog(ERROR, "Cannot write block %u of \"%s\": %s", + blknum, to_fullpath, strerror(errno)); + } + preWriteOff = 0; + targetSize = file->write_size - blknum * BLCKSZ; + } + } } else { - if (fio_fwrite(out, page.data, BLCKSZ) != BLCKSZ) - elog(ERROR, "Cannot write block %u of \"%s\": %s", - blknum, to_fullpath, strerror(errno)); + if (is_compressed) + { + ssize_t rc; + rc = fio_fwrite_compressed(out, page.data, compressed_size, file->compress_alg, NULL, NULL, NULL, NULL); + + if (!fio_is_remote_file(out) && rc != BLCKSZ) + { + pg_free(preWriteBuf); + elog(ERROR, "Cannot write block %u of \"%s\": %s, size: %u", + blknum, to_fullpath, strerror(errno), compressed_size); + } + } + else + { + if (fio_fwrite(out, page.data, BLCKSZ) != BLCKSZ) + { + pg_free(preWriteBuf); + elog(ERROR, "Cannot write block %u of \"%s\": %s", + blknum, to_fullpath, strerror(errno)); + } + } } write_len += BLCKSZ; @@ -1229,8 +1373,8 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers if (map) datapagemap_add(map, blknum); } - + pg_free(preWriteBuf); return write_len; } @@ -1688,6 +1832,11 @@ check_data_file(ConnectionArgs *arguments, pgFile *file, */ nblocks = file->size/BLCKSZ; + PreReadBuf preReadBuf; + preReadBuf.num = -1; + preReadBuf.data = (char*)malloc(DSS_BLCKSZ); + if (preReadBuf.data == NULL) + elog(ERROR, "malloc preReadBuf.data failed, size : %d", DSS_BLCKSZ); for (blknum = 0; blknum < nblocks; blknum++) { PageState page_st; @@ -1695,7 +1844,7 @@ check_data_file(ConnectionArgs *arguments, pgFile *file, page_state = prepare_page(NULL, file, InvalidXLogRecPtr, blknum, in, BACKUP_MODE_FULL, curr_page, false, checksum_version, - from_fullpath, &page_st, NULL, read_len); + from_fullpath, &page_st, NULL, read_len, &preReadBuf); if (page_state == PageIsTruncated) break; @@ -1710,6 +1859,7 @@ check_data_file(ConnectionArgs *arguments, pgFile *file, } } + pg_free(preReadBuf.data); fclose(in); return is_valid; } @@ -2222,6 +2372,11 @@ send_pages(ConnectionArgs* conn_arg, const char *to_fullpath, const char *from_f setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); } + PreReadBuf preReadBuf; + preReadBuf.num = -1; + preReadBuf.data = (char*)malloc(DSS_BLCKSZ); + if (preReadBuf.data == NULL) + elog(ERROR, "malloc preReadBuf.data failed, size : %d", DSS_BLCKSZ); while (blknum < (BlockNumber)file->n_blocks) { PageState page_st; @@ -2229,7 +2384,7 @@ send_pages(ConnectionArgs* conn_arg, const char *to_fullpath, const char *from_f int rc = prepare_page(conn_arg, file, prev_backup_start_lsn, blknum, in, backup_mode, curr_page, true, checksum_version, - from_fullpath, &page_st, pageCompression, read_len); + from_fullpath, &page_st, pageCompression, read_len, &preReadBuf); if (rc == PageIsTruncated) break; @@ -2277,6 +2432,7 @@ send_pages(ConnectionArgs* conn_arg, const char *to_fullpath, const char *from_f else blknum++; } + pg_free(preReadBuf.data); /* * Add dummy header, so we can later extract the length of last header diff --git a/src/bin/pg_probackup/file.cpp b/src/bin/pg_probackup/file.cpp index cc0f4d55b..8a4cb3389 100644 --- a/src/bin/pg_probackup/file.cpp +++ b/src/bin/pg_probackup/file.cpp @@ -633,7 +633,7 @@ int fio_truncate(int fd, off_t size) /* * Read file from specified location. */ -int fio_pread(FILE* f, void* buf, off_t offs, PageCompression* pageCompression) +int fio_pread(FILE* f, void* buf, off_t offs, PageCompression* pageCompression, int size) { if (fio_is_remote_file(f)) { @@ -660,13 +660,13 @@ int fio_pread(FILE* f, void* buf, off_t offs, PageCompression* pageCompression) { /* For local file, opened by fopen, we should use stdio functions */ if (pageCompression) { - return (int)pageCompression->ReadCompressedBuffer((BlockNumber)(offs / BLCKSZ), (char*)buf, BLCKSZ, true); + return (int)pageCompression->ReadCompressedBuffer((BlockNumber)(offs / BLCKSZ), (char*)buf, size, true); } else { int rc = fseek(f, offs, SEEK_SET); if (rc < 0) { return rc; } - return fread(buf, 1, BLCKSZ, f); + return fread(buf, 1, size, f); } } } @@ -765,7 +765,7 @@ fio_decompress(void* dst, void const* src, size_t size, int compress_alg) } /* Write data to the file */ -ssize_t fio_fwrite_compressed(FILE* f, void const* buf, size_t size, int compress_alg) +ssize_t fio_fwrite_compressed(FILE* f, void const* buf, size_t size, int compress_alg, const char *to_fullpath, char *preWriteBuf, int *preWriteOff, int *targetSize) { if (fio_is_remote_file(f)) { @@ -783,13 +783,37 @@ ssize_t fio_fwrite_compressed(FILE* f, void const* buf, size_t size, int compres } else { - /* operate is same in local mode and dss mode */ - char uncompressed_buf[BLCKSZ]; - int32 uncompressed_size = fio_decompress(uncompressed_buf, buf, size, compress_alg); - - return (uncompressed_size < 0) - ? uncompressed_size - : fio_fwrite(f, uncompressed_buf, uncompressed_size); + if (preWriteBuf != NULL) + { + int32 uncompressed_size = fio_decompress(preWriteBuf + (*preWriteOff), buf, size, compress_alg); + *preWriteOff += uncompressed_size; + if (*preWriteOff > DSS_BLCKSZ) + { + pg_free(preWriteBuf); + elog(ERROR, "Offset %d is bigger than preWriteBuf size %d", *preWriteOff, DSS_BLCKSZ); + } + if (*preWriteOff == DSS_BLCKSZ) + { + int write_len = fio_fwrite(f, preWriteBuf, DSS_BLCKSZ); + if (write_len != DSS_BLCKSZ) + { + pg_free(preWriteBuf); + elog(ERROR, "Cannot write block of \"%s\": %s, size: %u", + to_fullpath, strerror(errno), DSS_BLCKSZ); + } + *preWriteOff = 0; + *targetSize -= DSS_BLCKSZ; + } + return uncompressed_size; + } + else + { + char uncompressed_buf[BLCKSZ]; + int32 uncompressed_size = fio_decompress(uncompressed_buf, buf, size, compress_alg); + return (uncompressed_size < 0) + ? uncompressed_size + : fio_fwrite(f, uncompressed_buf, uncompressed_size); + } } } diff --git a/src/bin/pg_probackup/file.h b/src/bin/pg_probackup/file.h index aa465e604..1719d492a 100644 --- a/src/bin/pg_probackup/file.h +++ b/src/bin/pg_probackup/file.h @@ -85,6 +85,7 @@ typedef enum #define IO_CHECK(cmd, size) do { int _rc = (cmd); if (_rc != (int)(size)) fio_error(_rc, size, __FILE__, __LINE__); } while (0) #define FILE_PERMISSIONS 0600 +#define DSS_BLCKSZ 2097152 //2M , In dss mode, the size of operation(read and write) from a DSS file is 2M typedef struct { @@ -112,9 +113,9 @@ extern int fio_get_agent_version(void); extern FILE* fio_fopen(char const* name, char const* mode, fio_location location); extern size_t fio_fwrite(FILE* f, void const* buf, size_t size); extern void fio_construct_compressed(void const* buf, size_t size); -extern ssize_t fio_fwrite_compressed(FILE* f, void const* buf, size_t size, int compress_alg); +extern ssize_t fio_fwrite_compressed(FILE* f, void const* buf, size_t size, int compress_alg, const char *to_fullpath, char *preWriteBuf, int *preWriteOff, int *targetSize); extern ssize_t fio_fread(FILE* f, void* buf, size_t size); -extern int fio_pread(FILE* f, void* buf, off_t offs, PageCompression* pageCompression); +extern int fio_pread(FILE* f, void* buf, off_t offs, PageCompression* pageCompression, int size); extern int fio_fprintf(FILE* f, char const* arg, ...);// pg_attribute_printf(2, 3); extern int fio_fflush(FILE* f); extern int fio_fseek(FILE* f, off_t offs); From b81c7ee5995543a42d47b5b600dc7abb8eaef238 Mon Sep 17 00:00:00 2001 From: "arcoalien@qq.com" Date: Tue, 19 Sep 2023 17:49:27 +0800 Subject: [PATCH 259/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E4=B8=BB=E5=A4=87?= =?UTF-8?q?=E5=80=92=E6=8D=A2=E5=90=8E,=E6=96=B0=E4=B8=BB=E7=94=B1?= =?UTF-8?q?=E4=BA=8Efd=E6=98=AF0,=E5=AF=BC=E8=87=B4=E8=AF=BB=E5=92=8Caio?= =?UTF-8?q?=E5=87=BA=E9=94=99=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/storage/smgr/segment/data_file.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gausskernel/storage/smgr/segment/data_file.cpp b/src/gausskernel/storage/smgr/segment/data_file.cpp index 6ea800893..a64f31a4c 100644 --- a/src/gausskernel/storage/smgr/segment/data_file.cpp +++ b/src/gausskernel/storage/smgr/segment/data_file.cpp @@ -237,7 +237,7 @@ SegPhysicalFile df_get_physical_file(SegLogicFile *sf, int sliceno, BlockNumber } SegmentCheck(sliceno < sf->file_num); - if (SS_STANDBY_MODE && sf->segfiles[sliceno].fd <= 0) { + if (ENABLE_DMS && sf->segfiles[sliceno].fd <= 0) { char *filename = slice_filename(sf->filename, sliceno); sf->segfiles[sliceno].fd = dv_open_file(filename, O_RDONLY | PG_BINARY, SEGMENT_FILE_MODE); pfree(filename); From e066d37129d6ab8f0137a6eef794330890fff223 Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Tue, 19 Sep 2023 18:58:52 +0800 Subject: [PATCH 260/304] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E5=8F=8C=E9=9B=86?= =?UTF-8?q?=E7=BE=A4=E5=88=87=E6=8D=A2=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../storage/access/transam/xlog.cpp | 182 ++++++++---------- src/include/knl/knl_thread.h | 1 + 2 files changed, 81 insertions(+), 102 deletions(-) diff --git a/src/gausskernel/storage/access/transam/xlog.cpp b/src/gausskernel/storage/access/transam/xlog.cpp index 13e63c2c9..0fd2e34ad 100755 --- a/src/gausskernel/storage/access/transam/xlog.cpp +++ b/src/gausskernel/storage/access/transam/xlog.cpp @@ -5339,9 +5339,15 @@ static XLogRecord *ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, in latestValidRecord = t_thrd.xlog_cxt.ReadRecPtr; latestRecordCrc = record->xl_crc; latestRecordLen = record->xl_tot_len; + if (SS_REPLICATION_DORADO_CLUSTER) { + t_thrd.xlog_cxt.ssXlogReadFailedTimes = 0; + } /* Great, got a record */ return record; } else { + if (SS_REPLICATION_DORADO_CLUSTER) { + t_thrd.xlog_cxt.ssXlogReadFailedTimes++; + } /* No valid record available from this source */ if (streamFailCount < XLOG_STREAM_READREC_MAXTRY) { streamFailCount++; @@ -8879,6 +8885,7 @@ void StartupXLOG(void) t_thrd.xlog_cxt.startup_processing = true; t_thrd.xlog_cxt.RedoDone = false; t_thrd.xlog_cxt.currentRetryTimes = 0; + t_thrd.xlog_cxt.ssXlogReadFailedTimes = 0; t_thrd.xlog_cxt.forceFinishHappened = false; g_instance.comm_cxt.predo_cxt.redoPf.recovery_done_ptr = 0; g_instance.comm_cxt.predo_cxt.redoPf.redo_done_time = 0; @@ -19353,7 +19360,6 @@ static int SSDoradoReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePt XLogRecPtr RecPtr = targetPagePtr; uint32 targetPageOff; bool processtrxn = false; - bool fetching_ckpt = readprivate->fetching_ckpt; bool randAccess = readprivate->randAccess; XLogCtlData *xlogctl = t_thrd.shemem_ptr_cxt.XLogCtl; XLogSegNo replayedSegNo; @@ -19467,12 +19473,6 @@ static int SSDoradoReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePt pg_memory_barrier(); if (SS_REPLICATION_MAIN_STANBY_NODE && WalRcvIsDone() && CheckForFailoverTrigger()) { - t_thrd.xlog_cxt.receivedUpto = GetWalRcvWriteRecPtr(NULL); - if (XLByteLT(RecPtr, t_thrd.xlog_cxt.receivedUpto)) { - /* wait xlog redo done */ - continue; - } - ProcTxnWorkLoad(true); ereport(LOG, (errmsg("RecPtr(%X/%X), receivedUpto(%X/%X).", (uint32)(RecPtr >> 32), (uint32)RecPtr, (uint32)(t_thrd.xlog_cxt.receivedUpto >> 32), @@ -19507,124 +19507,97 @@ static int SSDoradoReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePt close(t_thrd.xlog_cxt.readFile); t_thrd.xlog_cxt.readFile = -1; } + t_thrd.xlog_cxt.failedSources = 0; /* Reset curFileTLI if random fetch. */ if (randAccess) { t_thrd.xlog_cxt.curFileTLI = 0; } - + if (SS_REPLICATION_MAIN_STANBY_NODE && CheckForFailoverTrigger()) { + goto triggered; + } /* * Try to restore the file from archive, or read an * existing file from pg_xlog. */ sources = XLOG_FROM_ARCHIVE | XLOG_FROM_PG_XLOG; - if (XLByteLE(t_thrd.xlog_cxt.receivedUpto, expectedRecPtr)) { - t_thrd.xlog_cxt.failedSources = 0; - /* - * Before we sleep, re-scan for possible new timelines - * if we were requested to recover to the latest - * timeline. - */ - if (t_thrd.xlog_cxt.recoveryTargetIsLatest) { - if (rescanLatestTimeLine()) { - continue; - } - } - - if (!xlogctl->IsRecoveryDone) { - g_instance.comm_cxt.predo_cxt.redoPf.redo_done_time = GetCurrentTimestamp(); - g_instance.comm_cxt.predo_cxt.redoPf.recovery_done_ptr = t_thrd.xlog_cxt.ReadRecPtr; - ereport(LOG, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), - errmsg("XLogPageRead IsRecoveryDone is set true, " - "ReadRecPtr:%X/%X, EndRecPtr:%X/%X, " - "receivedUpto:%X/%X, CtlInfo_insertHead:%X/%X.", - (uint32)(t_thrd.xlog_cxt.ReadRecPtr >> 32), - (uint32)(t_thrd.xlog_cxt.ReadRecPtr), - (uint32)(t_thrd.xlog_cxt.EndRecPtr >> 32), - (uint32)(t_thrd.xlog_cxt.EndRecPtr), - (uint32)(t_thrd.xlog_cxt.receivedUpto >> 32), - (uint32)(t_thrd.xlog_cxt.receivedUpto), - (uint32)(g_instance.xlog_cxt.ssReplicationXLogCtl->insertHead >> 32), - (uint32)(g_instance.xlog_cxt.ssReplicationXLogCtl->insertHead)))); - parallel_recovery::redo_dump_all_stats(); - } + if (t_thrd.startup_cxt.shutdown_requested) { + ereport(LOG, (errmsg("startup shutdown"))); + proc_exit(0); + } - /* - * signal postmaster to update local redo end - * point to gaussdb state file. - */ - ProcTxnWorkLoad(true); - if (!xlogctl->IsRecoveryDone) { - SendPostmasterSignal(PMSIGNAL_LOCAL_RECOVERY_DONE); - if (SS_PERFORMING_SWITCHOVER) { - g_instance.dms_cxt.SSClusterState = NODESTATE_STANDBY_PROMOTED; - } + /* + * Before we sleep, re-scan for possible new timelines + * if we were requested to recover to the latest + * timeline. + */ + if (t_thrd.xlog_cxt.recoveryTargetIsLatest) { + if (rescanLatestTimeLine()) { + continue; } + } - SpinLockAcquire(&xlogctl->info_lck); - xlogctl->IsRecoveryDone = true; - SpinLockRelease(&xlogctl->info_lck); - /* - * If primary_conninfo is set, launch walreceiver to - * try to stream the missing WAL, before retrying to - * restore from archive/pg_xlog. - * - * If fetching_ckpt is TRUE, RecPtr points to the - * initial checkpoint location. In that case, we use - * RedoStartLSN as the streaming start position - * instead of RecPtr, so that when we later jump - * backwards to start redo at RedoStartLSN, we will - * have the logs streamed already. - */ - load_server_mode(); + if (!xlogctl->IsRecoveryDone) { + g_instance.comm_cxt.predo_cxt.redoPf.redo_done_time = GetCurrentTimestamp(); + g_instance.comm_cxt.predo_cxt.redoPf.recovery_done_ptr = t_thrd.xlog_cxt.ReadRecPtr; + ereport(LOG, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), + errmsg("XLogPageRead IsRecoveryDone is set true, " + "ReadRecPtr:%X/%X, EndRecPtr:%X/%X, " + "receivedUpto:%X/%X, CtlInfo_insertHead:%X/%X.", + (uint32)(t_thrd.xlog_cxt.ReadRecPtr >> 32), + (uint32)(t_thrd.xlog_cxt.ReadRecPtr), + (uint32)(t_thrd.xlog_cxt.EndRecPtr >> 32), + (uint32)(t_thrd.xlog_cxt.EndRecPtr), + (uint32)(t_thrd.xlog_cxt.receivedUpto >> 32), + (uint32)(t_thrd.xlog_cxt.receivedUpto), + (uint32)(g_instance.xlog_cxt.ssReplicationXLogCtl->insertHead >> 32), + (uint32)(g_instance.xlog_cxt.ssReplicationXLogCtl->insertHead)))); + parallel_recovery::redo_dump_all_stats(); + } - if (SS_REPLICATION_MAIN_STANBY_NODE && CheckForFailoverTrigger()) { - goto triggered; + /* + * signal postmaster to update local redo end + * point to gaussdb state file. + */ + ProcTxnWorkLoad(true); + if (!xlogctl->IsRecoveryDone) { + SendPostmasterSignal(PMSIGNAL_LOCAL_RECOVERY_DONE); + if (SS_PERFORMING_SWITCHOVER) { + g_instance.dms_cxt.SSClusterState = NODESTATE_STANDBY_PROMOTED; } - ProcTxnWorkLoad(false); + } + + SpinLockAcquire(&xlogctl->info_lck); + xlogctl->IsRecoveryDone = true; + SpinLockRelease(&xlogctl->info_lck); + /* + * If primary_conninfo is set, launch walreceiver to + * try to stream the missing WAL, before retrying to + * restore from archive/pg_xlog. + * + * If fetching_ckpt is TRUE, RecPtr points to the + * initial checkpoint location. In that case, we use + * RedoStartLSN as the streaming start position + * instead of RecPtr, so that when we later jump + * backwards to start redo at RedoStartLSN, we will + * have the logs streamed already. + */ + load_server_mode(); + ProcTxnWorkLoad(false); + if (!SS_IN_REFORM) { /* use volatile pointer to prevent code rearrangement */ volatile WalRcvData *walrcv = t_thrd.walreceiverfuncs_cxt.WalRcv; rename_recovery_conf_for_roach(); ereport(LOG, (errmsg("request xlog stream from dorado copy at %X/%X.", - fetching_ckpt ? (uint32)(t_thrd.xlog_cxt.RedoStartLSN >> 32) - : (uint32)(targetRecPtr >> 32), - fetching_ckpt ? (uint32)t_thrd.xlog_cxt.RedoStartLSN - : (uint32)targetRecPtr))); + (uint32)(targetRecPtr >> 32), + (uint32)targetRecPtr))); ShutdownWalRcv(); SpinLockAcquire(&walrcv->mutex); walrcv->receivedUpto = 0; SpinLockRelease(&walrcv->mutex); - RequestXLogStreaming(fetching_ckpt ? &t_thrd.xlog_cxt.RedoStartLSN : &targetRecPtr, 0, - REPCONNTARGET_SHARED_STORAGE, 0); - continue; - } - - if (CheckForFailoverTrigger()) { - XLogRecPtr receivedUpto = GetWalRcvWriteRecPtr(NULL); - XLogRecPtr EndRecPtrTemp = t_thrd.xlog_cxt.EndRecPtr; - XLByteAdvance(EndRecPtrTemp, SizeOfXLogRecord); - if (XLByteLT(EndRecPtrTemp, receivedUpto) && !FORCE_FINISH_ENABLED && - t_thrd.xlog_cxt.currentRetryTimes++ < g_retryTimes) { - ereport(WARNING, (errmsg("there are some received xlog have not been redo " - "the tail of last redo lsn:%X/%X, received lsn:%X/%X, retry %d times", - (uint32)(EndRecPtrTemp >> 32), (uint32)EndRecPtrTemp, - (uint32)(receivedUpto >> 32), (uint32)receivedUpto, - t_thrd.xlog_cxt.currentRetryTimes))); - return -1; - } - ereport(LOG, - (errmsg("read record failed when promoting, current lsn (%X/%X), received lsn(%X/%X)," - "sources[%u], failedSources[%u], readSource[%u], readFile[%d], readId[%u]," - "readSeg[%u], readOff[%u], readLen[%u]", - (uint32)(RecPtr >> 32), (uint32)RecPtr, - (uint32)(t_thrd.xlog_cxt.receivedUpto >> 32), - (uint32)t_thrd.xlog_cxt.receivedUpto, sources, t_thrd.xlog_cxt.failedSources, - t_thrd.xlog_cxt.readSource, t_thrd.xlog_cxt.readFile, - (uint32)(t_thrd.xlog_cxt.readSegNo >> 32), (uint32)t_thrd.xlog_cxt.readSegNo, - t_thrd.xlog_cxt.readOff, t_thrd.xlog_cxt.readLen))); - goto triggered; + RequestXLogStreaming(&targetRecPtr, 0, REPCONNTARGET_SHARED_STORAGE, 0); } break; } @@ -19647,10 +19620,11 @@ static int SSDoradoReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePt } } +needread: if (t_thrd.xlog_cxt.readFile < 0) { t_thrd.xlog_cxt.readFile = SSXLogFileOpenAnyTLI(t_thrd.xlog_cxt.readSegNo, emode, sources, xlog_path); if (t_thrd.xlog_cxt.readFile < 0) { - ereport(LOG, (errmsg("%s read failed, change xlog file.", xlog_path))); + ereport(LOG, (errmsg("%s open failed, change xlog file.", xlog_path))); goto next_record_is_invalid; } } @@ -19679,7 +19653,11 @@ static int SSDoradoReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePt t_thrd.xlog_cxt.readSource = 0; return -1; + triggered: + if (t_thrd.xlog_cxt.ssXlogReadFailedTimes < XLOG_STREAM_READREC_MAXTRY) { + goto needread; + } if (t_thrd.xlog_cxt.readFile >= 0) { close(t_thrd.xlog_cxt.readFile); } @@ -19687,7 +19665,7 @@ static int SSDoradoReadXLog(XLogReaderState *xlogreader, XLogRecPtr targetPagePt t_thrd.xlog_cxt.readLen = 0; t_thrd.xlog_cxt.readSource = 0; t_thrd.xlog_cxt.recoveryTriggered = true; - + return -1; } diff --git a/src/include/knl/knl_thread.h b/src/include/knl/knl_thread.h index 088c655bb..187177f9f 100755 --- a/src/include/knl/knl_thread.h +++ b/src/include/knl/knl_thread.h @@ -782,6 +782,7 @@ typedef struct knl_t_xlog_context { /* for switchover failed when load xlog record invalid retry count */ int currentRetryTimes; RedoTimeCost timeCost[TIME_COST_NUM]; + int ssXlogReadFailedTimes; } knl_t_xlog_context; typedef struct knl_t_dfs_context { From 23eb4657ab939f4c629a3e8de4a6cded80b1cfe6 Mon Sep 17 00:00:00 2001 From: zhang_xubo <2578876417@qq.com> Date: Tue, 19 Sep 2023 19:24:43 +0800 Subject: [PATCH 261/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D3.0.0-3.1.0-5.1.0?= =?UTF-8?q?=E8=B7=A8=E7=89=88=E6=9C=AC=E7=81=B0=E5=BA=A6=E5=8D=87=E7=BA=A7?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5=EF=BC=8C=E6=8A=A5=E9=94=99Key=20(oid)=3D(560?= =?UTF-8?q?)=20already=20exists?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../upgrade-post_catalog_maindb_92_806.sql | 2 ++ .../upgrade-post_catalog_otherdb_92_806.sql | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_806.sql b/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_806.sql index 1b19742eb..706cf5506 100644 --- a/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_806.sql +++ b/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_806.sql @@ -18,6 +18,8 @@ comment on function PG_CATALOG.sha1(text) is 'use the sha1 algorithm to hash'; -- sha2 SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 560; +DROP FUNCTION IF EXISTS pg_catalog.sha2(text, bigint) CASCADE; +DROP FUNCTION IF EXISTS pg_catalog.sha2(text, int) CASCADE; CREATE OR REPLACE FUNCTION pg_catalog.sha2(text, bigint) RETURNS text LANGUAGE internal diff --git a/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_806.sql b/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_806.sql index 1b19742eb..706cf5506 100644 --- a/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_806.sql +++ b/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_806.sql @@ -18,6 +18,8 @@ comment on function PG_CATALOG.sha1(text) is 'use the sha1 algorithm to hash'; -- sha2 SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 560; +DROP FUNCTION IF EXISTS pg_catalog.sha2(text, bigint) CASCADE; +DROP FUNCTION IF EXISTS pg_catalog.sha2(text, int) CASCADE; CREATE OR REPLACE FUNCTION pg_catalog.sha2(text, bigint) RETURNS text LANGUAGE internal From d5dcb822d13dc6abacd914ff8b3c4118730aa56a Mon Sep 17 00:00:00 2001 From: muyulinzhong Date: Mon, 18 Sep 2023 19:23:47 +0800 Subject: [PATCH 262/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=20=E3=80=90=E4=BD=BF?= =?UTF-8?q?=E7=94=A8alter=20system=E6=96=B9=E5=BC=8F=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E5=8F=82=E6=95=B0num=5Finternal=5Flock=5Fpartitions=E5=90=8E?= =?UTF-8?q?=EF=BC=8C=E9=87=8D=E5=90=AF=E6=95=B0=E6=8D=AE=E5=BA=93=E5=A4=B1?= =?UTF-8?q?=E8=B4=A5=E3=80=91=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../backend/utils/misc/guc/guc_storage.cpp | 23 +++++++++++++++---- src/gausskernel/optimizer/commands/copy.cpp | 14 ++++++++++- .../access/transam/multi_redo_settings.cpp | 4 ++-- .../storage/replication/logical/logical.cpp | 9 ++++++++ src/include/commands/copy.h | 2 +- src/include/replication/logical.h | 2 +- 6 files changed, 44 insertions(+), 10 deletions(-) diff --git a/src/common/backend/utils/misc/guc/guc_storage.cpp b/src/common/backend/utils/misc/guc/guc_storage.cpp index 3c65f8849..1ee3c7c7f 100755 --- a/src/common/backend/utils/misc/guc/guc_storage.cpp +++ b/src/common/backend/utils/misc/guc/guc_storage.cpp @@ -5019,7 +5019,7 @@ void InitializeNumLwLockPartitions(void) /* set default values */ SetLWLockPartDefaultNum(); /* Do str copy and remove space. */ - char* attr = TrimStr(g_instance.attr.attr_storage.num_internal_lock_partitions_str); + char* attr = TrimStrQuote(g_instance.attr.attr_storage.num_internal_lock_partitions_str, true); if (attr == NULL || attr[0] == '\0') { /* use default values */ return; } @@ -6504,11 +6504,24 @@ static void assign_ss_log_backup_file_count(int newval, void *extra) static bool check_logical_decode_options_default(char** newval, void** extra, GucSource source) { - if (!LogicalDecodeParseOptionsDefault(*newval, extra)) { - GUC_check_errdetail("invalid parameter setting for loglical_decode_options_default"); - return false; + /*Check argument whether coming frmo SYATEM ALTER SET*/ + char* temp = *newval; + int len = strlen(temp); + char ch = (len > 0) ? temp[len-1] : '\0'; + if(QuoteCheckOut(temp)) { + temp[len - 1] = '\0'; + temp++; + } + if (!LogicalDecodeParseOptionsDefault(temp, extra)) { + GUC_check_errdetail("invalid parameter setting for loglical_decode_options_default"); + if(len != 0) { + (*newval)[len - 1] = ch; + } + return false; + } + if(len != 0) { + (*newval)[len - 1] = ch; } - return true; } diff --git a/src/gausskernel/optimizer/commands/copy.cpp b/src/gausskernel/optimizer/commands/copy.cpp index e64d5de5a..8d8061df3 100644 --- a/src/gausskernel/optimizer/commands/copy.cpp +++ b/src/gausskernel/optimizer/commands/copy.cpp @@ -8777,7 +8777,7 @@ bool StrToInt32(const char* s, int *val) return true; } -char* TrimStr(const char* str) +char* TrimStrQuote(const char* str, bool isQuote) { if (str == NULL) { return NULL; @@ -8805,12 +8805,24 @@ char* TrimStr(const char* str) } len = end - begin + 1; + + if (isQuote && len>=2 && *begin == '"' && *end == '"') { + begin++; + end--; + len = end - begin + 1; + } + rc = memmove_s(cpyStr, strlen(cpyStr), begin, len); securec_check(rc, "\0", "\0"); cpyStr[len] = '\0'; return cpyStr; } +char* TrimStr(const char* str) +{ + return TrimStrQuote(str, false); +} + /* Deserialize the LOCATION options into locations list. * the multi-locations should be separated by '|' */ diff --git a/src/gausskernel/storage/access/transam/multi_redo_settings.cpp b/src/gausskernel/storage/access/transam/multi_redo_settings.cpp index d0007c8fc..f92a3fe20 100644 --- a/src/gausskernel/storage/access/transam/multi_redo_settings.cpp +++ b/src/gausskernel/storage/access/transam/multi_redo_settings.cpp @@ -29,7 +29,7 @@ #include "postgres.h" #include "knl/knl_variable.h" #include "utils/guc.h" - +#include "commands/copy.h" #include "access/multi_redo_settings.h" #include "access/multi_redo_api.h" #include "threadpool/threadpool_controler.h" @@ -133,7 +133,7 @@ static uint32 GetCPUCount() void ParseBindCpuInfo(RedoCpuBindControl *control) { - char* attr = TrimStr(g_instance.attr.attr_storage.redo_bind_cpu_attr); + char* attr = TrimStrQuote(g_instance.attr.attr_storage.redo_bind_cpu_attr, true); if (attr == NULL) { return; } diff --git a/src/gausskernel/storage/replication/logical/logical.cpp b/src/gausskernel/storage/replication/logical/logical.cpp index e025143cb..ea6ecf772 100644 --- a/src/gausskernel/storage/replication/logical/logical.cpp +++ b/src/gausskernel/storage/replication/logical/logical.cpp @@ -85,6 +85,15 @@ static void parallel_change_cb_wrapper(ParallelReorderBuffer *cache, ReorderBuff static void LoadOutputPlugin(OutputPluginCallbacks *callbacks, const char *plugin); static void LoadOutputPlugin(ParallelOutputPluginCallbacks *callbacks, const char *plugin); +/* Checkout aurgments whether coming from ALTER SYSTEM SET*/ +bool QuoteCheckOut(char* newval) +{ + int len = strlen(newval); + if(len >= 2 && newval[0] == '"' && newval[0] == newval[len - 1]) + return true; + return false; +} + /* * Make sure the current settings & environment are capable of doing logical * decoding. diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h index c9436f62f..f9ef63920 100644 --- a/src/include/commands/copy.h +++ b/src/include/commands/copy.h @@ -451,8 +451,8 @@ extern int GetDecimalFromHex(char hex); extern char* limit_printout_length(const char* str); extern bool StrToInt32(const char* s, int *val); +extern char* TrimStrQuote(const char* str, bool isQuote); extern char* TrimStr(const char* str); - extern void UHeapAddToBulkInsertSelect(CopyFromBulk bulk, Tuple tup, bool needCopy); extern void HeapAddToBulkInsertSelect(CopyFromBulk bulk, Tuple tup, bool needCopy); diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h index 9b9e44e00..015552c67 100644 --- a/src/include/replication/logical.h +++ b/src/include/replication/logical.h @@ -306,7 +306,7 @@ typedef struct DecodeOptionsDefault { extern LogicalDispatcher g_Logicaldispatcher[]; extern bool firstCreateDispatcher; - +extern bool QuoteCheckOut(char* newval); extern void CheckLogicalDecodingRequirements(Oid databaseId); extern void ParallelReorderBufferQueueChange(ParallelReorderBuffer *rb, logicalLog *change, int slotId); extern void ParallelReorderBufferForget(ParallelReorderBuffer *rb, int slotId, ParallelReorderBufferTXN *txn); From 23adb56de34b2aa166502883eceb4a1c6f3a9f94 Mon Sep 17 00:00:00 2001 From: totaj Date: Tue, 19 Sep 2023 22:33:19 +0800 Subject: [PATCH 263/304] Job schedule should load dolphin. --- src/common/backend/utils/init/postinit.cpp | 2 ++ src/gausskernel/process/job/job_scheduler.cpp | 3 +++ 2 files changed, 5 insertions(+) diff --git a/src/common/backend/utils/init/postinit.cpp b/src/common/backend/utils/init/postinit.cpp index 8a8ea27b9..5c4d74743 100644 --- a/src/common/backend/utils/init/postinit.cpp +++ b/src/common/backend/utils/init/postinit.cpp @@ -1254,6 +1254,8 @@ void PostgresInitializer::InitJobScheduler() InitSettings(); + InitExtensionVariable(); + FinishInit(); } diff --git a/src/gausskernel/process/job/job_scheduler.cpp b/src/gausskernel/process/job/job_scheduler.cpp index 3f6073001..a014c7bfe 100755 --- a/src/gausskernel/process/job/job_scheduler.cpp +++ b/src/gausskernel/process/job/job_scheduler.cpp @@ -201,6 +201,9 @@ NON_EXEC_STATIC void JobScheduleMain() /* Initialize openGauss with DEFAULT_DATABASE, since it cannot be dropped */ t_thrd.proc_cxt.PostInit->SetDatabaseAndUser(dbname, InvalidOid, username); t_thrd.proc_cxt.PostInit->InitJobScheduler(); +#if (!defined(ENABLE_MULTIPLE_NODES)) && (!defined(ENABLE_PRIVATEGAUSS)) + LoadSqlPlugin(); +#endif #ifdef PGXC /* PGXC_COORD */ /* From 5d8575c650d837a0567c8d58a13491acd5d24adb Mon Sep 17 00:00:00 2001 From: yanghao Date: Wed, 20 Sep 2023 10:09:53 +0800 Subject: [PATCH 264/304] adapt input and output for gb18030-2022 --- src/gausskernel/optimizer/commands/copy.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/gausskernel/optimizer/commands/copy.cpp b/src/gausskernel/optimizer/commands/copy.cpp index e64d5de5a..546269863 100644 --- a/src/gausskernel/optimizer/commands/copy.cpp +++ b/src/gausskernel/optimizer/commands/copy.cpp @@ -1251,7 +1251,8 @@ void VerifyEncoding(int encoding) { Oid proc; - if (encoding == GetDatabaseEncoding() || encoding == PG_SQL_ASCII || GetDatabaseEncoding() == PG_SQL_ASCII) + if (encoding == GetDatabaseEncoding() || encoding == PG_SQL_ASCII || GetDatabaseEncoding() == PG_SQL_ASCII || + (GetDatabaseEncoding() == PG_GB18030_2022 && encoding == PG_GB18030)) return; proc = FindDefaultConversionProc(encoding, GetDatabaseEncoding()); From e1a8b451176edea6ca339f0e27ceaf6a47afcf8b Mon Sep 17 00:00:00 2001 From: congzhou2603 Date: Tue, 19 Sep 2023 10:43:51 +0800 Subject: [PATCH 265/304] =?UTF-8?q?=E3=80=90bugfix=E3=80=91=20=E4=BF=AE?= =?UTF-8?q?=E5=A4=8Dpostmaster=E7=BA=BF=E7=A8=8B=E5=8D=A1=E5=9C=A8PM=5FWAI?= =?UTF-8?q?T=5FBACKEND?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../process/postmaster/postmaster.cpp | 22 +++++++++++++++++++ src/include/ddes/dms/ss_reform_common.h | 2 ++ 2 files changed, 24 insertions(+) diff --git a/src/gausskernel/process/postmaster/postmaster.cpp b/src/gausskernel/process/postmaster/postmaster.cpp index d8b0ff25e..706a785dc 100644 --- a/src/gausskernel/process/postmaster/postmaster.cpp +++ b/src/gausskernel/process/postmaster/postmaster.cpp @@ -486,6 +486,7 @@ static bool CheckSignalByFile(const char *filename, void *infoPtr, size_t infoSi int GaussDbThreadMain(knl_thread_arg* arg); const char* GetThreadName(knl_thread_role role); +void SSOndemandProcExitIfStayWaitBackends(); #ifdef EXEC_BACKEND @@ -3976,6 +3977,10 @@ static int ServerLoop(void) result = BackendStartup(port, isConnectHaPort); } + if (SS_CLUSTER_ONDEMAND_RECOVERY && SS_IN_REFORM && + result != STATUS_OK && pmState == PM_WAIT_BACKENDS) { + SSOndemandProcExitIfStayWaitBackends(); + } if (result != STATUS_OK) { if (port->is_logic_conn) { gs_close_gsocket(&port->gs_sock); @@ -15086,3 +15091,20 @@ void SSRestartFailoverPromote() pmState = PM_WAIT_BACKENDS; SShandle_promote_signal(); } + +void SSOndemandProcExitIfStayWaitBackends() +{ + int failTimes = 0; + while (failTimes < WAIT_PMSTATE_UPDATE_TRIES && pmState == PM_WAIT_BACKENDS) { + PostmasterStateMachine(); + pg_usleep(REFORM_WAIT_LONG); + failTimes++; + } + if (pmState == PM_WAIT_BACKENDS) { + ereport(PANIC, (errmsg("Proc exit because pmState stay %s for %d times, " + "when reform failed and in ondemand recovery, " + "to avoid pmState being stuck in PM_WAIT_BACKENDS.", + GetPMState(pmState), WAIT_PMSTATE_UPDATE_TRIES))); + proc_exit(1); + } +} \ No newline at end of file diff --git a/src/include/ddes/dms/ss_reform_common.h b/src/include/ddes/dms/ss_reform_common.h index 227d762e8..a11429d1d 100644 --- a/src/include/ddes/dms/ss_reform_common.h +++ b/src/include/ddes/dms/ss_reform_common.h @@ -31,6 +31,8 @@ #define REFORM_WAIT_LONG 100000 /* 0.1 sec */ #define WAIT_REFORM_CTRL_REFRESH_TRIES 1000 +#define WAIT_PMSTATE_UPDATE_TRIES 100 + #define REFORM_CTRL_VERSION 1 typedef struct SSBroadcastCancelTrx { From fab323acdee49049112b0a29eed31ce5081225ee Mon Sep 17 00:00:00 2001 From: jiangyan300 Date: Wed, 20 Sep 2023 21:02:20 +0800 Subject: [PATCH 266/304] =?UTF-8?q?memcheck=E5=8C=85=E6=89=A7=E8=A1=8Cci?= =?UTF-8?q?=E7=94=A8=E4=BE=8B=EF=BC=8Cjson.cpp=E4=B8=ADjson=5Fbuild=5Farra?= =?UTF-8?q?y=E5=87=BD=E6=95=B0=E4=B8=AD=E5=AD=98=E5=9C=A8=E6=BA=A2?= =?UTF-8?q?=E5=87=BA=E9=A3=8E=E9=99=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit memcheck包执行ci用例，json.cpp中json_build_array函数中存在溢出风险 --- src/common/backend/utils/adt/json.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/common/backend/utils/adt/json.cpp b/src/common/backend/utils/adt/json.cpp index ba12288de..47d40d1d3 100644 --- a/src/common/backend/utils/adt/json.cpp +++ b/src/common/backend/utils/adt/json.cpp @@ -1892,7 +1892,6 @@ Datum json_build_array(PG_FUNCTION_ARGS) appendStringInfoChar(result, '['); for (i = 0; i < nargs; i++) { val_type = get_fn_expr_argtype(fcinfo->flinfo, i); - arg = PG_GETARG_DATUM(i + 1); /* see comments in json_build_object above */ if (val_type == UNKNOWNOID && get_fn_expr_arg_stable(fcinfo->flinfo, i)) { val_type = TEXTOID; From f6f95e9995f37e77a1c56ebc1f01ba5c0091eed0 Mon Sep 17 00:00:00 2001 From: congzhou2603 Date: Wed, 20 Sep 2023 16:09:11 +0800 Subject: [PATCH 267/304] =?UTF-8?q?=E3=80=90bugfix=E3=80=91=E4=BF=AE?= =?UTF-8?q?=E6=94=B9=E6=8C=89=E9=9C=80=E5=9B=9E=E6=94=BE=E8=B5=B0=E5=88=B0?= =?UTF-8?q?=E6=9E=81=E8=87=B4RTO=E5=A4=87=E6=9C=BA=E5=8F=AF=E8=AF=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/process/postmaster/postmaster.cpp | 4 ++-- src/gausskernel/storage/buffer/bufmgr.cpp | 7 +++---- src/include/access/multi_redo_api.h | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/gausskernel/process/postmaster/postmaster.cpp b/src/gausskernel/process/postmaster/postmaster.cpp index 706a785dc..2a4e68afb 100644 --- a/src/gausskernel/process/postmaster/postmaster.cpp +++ b/src/gausskernel/process/postmaster/postmaster.cpp @@ -3979,7 +3979,7 @@ static int ServerLoop(void) if (SS_CLUSTER_ONDEMAND_RECOVERY && SS_IN_REFORM && result != STATUS_OK && pmState == PM_WAIT_BACKENDS) { - SSOndemandProcExitIfStayWaitBackends(); + SSOndemandProcExitIfStayWaitBackends(); } if (result != STATUS_OK) { if (port->is_logic_conn) { @@ -15101,7 +15101,7 @@ void SSOndemandProcExitIfStayWaitBackends() failTimes++; } if (pmState == PM_WAIT_BACKENDS) { - ereport(PANIC, (errmsg("Proc exit because pmState stay %s for %d times, " + ereport(WARNING, (errmsg("Proc exit because pmState stay %s for %d times, " "when reform failed and in ondemand recovery, " "to avoid pmState being stuck in PM_WAIT_BACKENDS.", GetPMState(pmState), WAIT_PMSTATE_UPDATE_TRIES))); diff --git a/src/gausskernel/storage/buffer/bufmgr.cpp b/src/gausskernel/storage/buffer/bufmgr.cpp index e807eb956..65601d55a 100644 --- a/src/gausskernel/storage/buffer/bufmgr.cpp +++ b/src/gausskernel/storage/buffer/bufmgr.cpp @@ -1714,11 +1714,10 @@ Buffer buffer_read_extended_internal(Relation reln, ForkNumber fork_num, BlockNu Buffer ReadBufferExtended(Relation reln, ForkNumber fork_num, BlockNumber block_num, ReadBufferMode mode, BufferAccessStrategy strategy) { - if (IsDefaultExtremeRtoMode() && - (!RecoveryInProgress() || !IsExtremeRtoRunning() || !is_exrto_standby_read_worker())) { - return buffer_read_extended_internal(reln, fork_num, block_num, mode, strategy); - } else { + if (IsDefaultExtremeRtoMode() && RecoveryInProgress() && IsExtremeRtoRunning() && is_exrto_standby_read_worker()) { return standby_read_buf(reln, fork_num, block_num, mode, strategy); + } else { + return buffer_read_extended_internal(reln, fork_num, block_num, mode, strategy); } } diff --git a/src/include/access/multi_redo_api.h b/src/include/access/multi_redo_api.h index 0c1bd1f77..4ec8cee78 100644 --- a/src/include/access/multi_redo_api.h +++ b/src/include/access/multi_redo_api.h @@ -68,7 +68,7 @@ static const uint32 BIG_RECORD_LENGTH = XLOG_BLCKSZ * 16; #define IS_EXRTO_READ (IsExtremeRedo() && g_instance.attr.attr_storage.EnableHotStandby && IsDefaultExtremeRtoMode()) #define IS_EXRTO_STANDBY_READ (IS_EXRTO_READ && pm_state_is_hot_standby()) -#define IS_EXRTO_RECOVERY_IN_PROGRESS (RecoveryInProgress() && IsExtremeRedo()) +#define IS_EXRTO_RECOVERY_IN_PROGRESS (RecoveryInProgress() && IsExtremeRedo() && IsDefaultExtremeRtoMode()) inline bool is_exrto_standby_read_worker() { From 47e77fa1843752636b8d9931c6e3c9f98ae22785 Mon Sep 17 00:00:00 2001 From: dongning12 Date: Thu, 21 Sep 2023 09:59:02 +0800 Subject: [PATCH 268/304] =?UTF-8?q?=E3=80=90=E8=B5=84=E6=BA=90=E6=B1=A0?= =?UTF-8?q?=E5=8C=96=E3=80=91=E5=AF=B9=E4=BA=8E=E8=AF=B7=E6=B1=82=E9=A1=B5?= =?UTF-8?q?=E9=9D=A2=E9=95=BF=E6=97=B6=E9=97=B4=E7=AD=89=E5=BE=85=E7=9A=84?= =?UTF-8?q?=E6=97=A5=E5=BF=97=EF=BC=8C=E9=81=BF=E5=85=8D=E5=85=B6=E6=89=93?= =?UTF-8?q?=E5=8D=B0=E5=88=B0=E5=89=8D=E7=AB=AFgsql?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/storage/buffer/bufmgr.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/gausskernel/storage/buffer/bufmgr.cpp b/src/gausskernel/storage/buffer/bufmgr.cpp index e807eb956..ffa42c040 100644 --- a/src/gausskernel/storage/buffer/bufmgr.cpp +++ b/src/gausskernel/storage/buffer/bufmgr.cpp @@ -5961,10 +5961,13 @@ void LockBuffer(Buffer buffer, int mode) long sleep_time = SSGetBufSleepTime(dms_retry_times); if (sleep_time == SS_BUF_MAX_WAIT_TIME && !SS_IN_REFORM) { volatile BufferTag *tag = &buf->tag; - ereport(WARNING, (errmodule(MOD_DMS), (errmsg("[SS buf][%u/%u/%u/%d %d-%u] request buf timeout, " + int output_backup = t_thrd.postgres_cxt.whereToSendOutput; + t_thrd.postgres_cxt.whereToSendOutput = DestNone; + ereport(WARNING, (errmodule(MOD_DMS), (errmsg("[SS buf][%u/%u/%u/%d %d-%u] LockBuffer, request buf timeout, " "buf_id:%d", tag->rnode.spcNode, tag->rnode.dbNode, tag->rnode.relNode, tag->rnode.bucketNode, tag->forkNum, tag->blockNum, buf->buf_id)))); + t_thrd.postgres_cxt.whereToSendOutput = output_backup; } pg_usleep(sleep_time); goto retry; @@ -6078,10 +6081,13 @@ bool ConditionalLockBuffer(Buffer buffer) long sleep_time = SSGetBufSleepTime(dms_retry_times); if (sleep_time == SS_BUF_MAX_WAIT_TIME && !SS_IN_REFORM) { volatile BufferTag *tag = &buf->tag; - ereport(WARNING, (errmodule(MOD_DMS), (errmsg("[SS buf][%u/%u/%u/%d %d-%u] request buf timeout, " + int output_backup = t_thrd.postgres_cxt.whereToSendOutput; + t_thrd.postgres_cxt.whereToSendOutput = DestNone; + ereport(WARNING, (errmodule(MOD_DMS), (errmsg("[SS buf][%u/%u/%u/%d %d-%u] ConditionalLockBuffer， request buf timeout, " "buf_id:%d", tag->rnode.spcNode, tag->rnode.dbNode, tag->rnode.relNode, tag->rnode.bucketNode, tag->forkNum, tag->blockNum, buf->buf_id)))); + t_thrd.postgres_cxt.whereToSendOutput = output_backup; } pg_usleep(sleep_time); goto retry; From bd4dd160b8eeec47460ccb4d9085388c98ca11cb Mon Sep 17 00:00:00 2001 From: openGaussDev Date: Tue, 12 Sep 2023 17:23:12 +0800 Subject: [PATCH 269/304] =?UTF-8?q?=E5=90=8C=E6=AD=A5=E6=9E=81=E8=87=B4RTO?= =?UTF-8?q?=E6=94=AF=E6=8C=81=E5=A4=87=E6=9C=BA=E8=AF=BB=E9=97=AE=E9=A2=98?= =?UTF-8?q?=E4=BF=AE=E5=A4=8D=20Offering:=20openGaussDev=20More=20detail:?= =?UTF-8?q?=E5=90=8C=E6=AD=A5=E6=9E=81=E8=87=B4RTO=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E5=A4=87=E6=9C=BA=E8=AF=BB=E9=97=AE=E9=A2=98=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/gs_guc/cluster_guc.conf | 1 + src/common/backend/catalog/storage.cpp | 6 +- src/common/backend/utils/errcodes.txt | 3 + src/common/backend/utils/init/globals.cpp | 3 +- src/common/backend/utils/misc/guc.cpp | 41 +- .../backend/utils/misc/guc/guc_storage.cpp | 38 +- src/common/backend/utils/time/snapmgr.cpp | 1 + .../optimizer/commands/tablespace.cpp | 10 +- .../process/postmaster/postmaster.cpp | 5 +- src/gausskernel/process/stream/streamMain.cpp | 16 +- src/gausskernel/process/tcop/postgres.cpp | 2 +- .../process/threadpool/knl_instance.cpp | 5 + .../process/threadpool/knl_thread.cpp | 1 + .../storage/access/common/reloptions.cpp | 2 +- .../storage/access/heap/heapam.cpp | 2 +- .../storage/access/nbtree/nbtpage.cpp | 33 +- .../storage/access/redo/redo_ginxlog.cpp | 17 +- .../storage/access/redo/redo_gistxlog.cpp | 2 + .../storage/access/redo/redo_heapam.cpp | 138 +----- .../storage/access/redo/redo_nbtxlog.cpp | 68 +-- .../storage/access/redo/redo_ubtxlog.cpp | 95 ++-- .../storage/access/redo/redo_xlogutils.cpp | 64 ++- .../storage/access/redo/standby_read/Makefile | 2 +- .../redo/standby_read/block_info_proc.cpp | 88 +++- .../redo/standby_read/lsn_info_proc.cpp | 174 ++++++- .../standby_read/standby_read_delay_ddl.cpp | 127 +++++- .../standby_read/standby_read_interface.cpp | 218 ++++++++- .../redo/standby_read/standby_read_proc.cpp | 423 +++++++++++++++++ .../storage/access/rmgrdesc/standbydesc.cpp | 51 ++- .../storage/access/transam/clog.cpp | 28 ++ .../access/transam/extreme_rto/dispatcher.cpp | 14 +- .../transam/extreme_rto/exrto_recycle.cpp | 8 +- .../access/transam/extreme_rto/page_redo.cpp | 429 ++++-------------- .../transam/parallel_recovery/dispatcher.cpp | 25 +- .../transam/parallel_recovery/page_redo.cpp | 1 - .../parallel_recovery/spsc_blocking_queue.cpp | 7 +- .../storage/access/transam/slru.cpp | 5 +- .../storage/access/transam/transam.cpp | 3 +- .../storage/access/transam/twophase_rmgr.cpp | 8 - .../storage/access/transam/xact.cpp | 4 +- .../storage/access/transam/xlog.cpp | 83 +++- .../storage/access/ubtree/ubtpage.cpp | 4 +- .../storage/access/ubtree/ubtsearch.cpp | 2 +- .../storage/access/ubtree/ubtutils.cpp | 4 + .../access/ustore/knl_uextremeredo.cpp | 16 +- .../storage/access/ustore/knl_uundorecord.cpp | 3 +- .../storage/access/ustore/knl_uvisibility.cpp | 8 +- .../access/ustore/undo/knl_uundorecycle.cpp | 45 +- .../access/ustore/undo/knl_uundospace.cpp | 2 +- .../access/ustore/undo/knl_uundozone.cpp | 8 +- src/gausskernel/storage/buffer/bufmgr.cpp | 2 - src/gausskernel/storage/ipc/procarray.cpp | 38 +- src/gausskernel/storage/ipc/sinvaladt.cpp | 15 +- src/gausskernel/storage/ipc/standby.cpp | 145 +++--- src/gausskernel/storage/lmgr/lock.cpp | 39 +- .../storage/replication/walreceiver.cpp | 1 + .../storage/smgr/storage_exrto_file.cpp | 56 ++- src/include/access/clog.h | 1 + src/include/access/extreme_rto/page_redo.h | 6 +- .../standby_read/block_info_meta.h | 10 +- .../extreme_rto/standby_read/lsn_info_meta.h | 2 + .../standby_read/standby_read_base.h | 6 + .../standby_read/standby_read_delay_ddl.h | 1 + src/include/access/multi_redo_api.h | 4 +- src/include/access/nbtree.h | 9 +- .../access/parallel_recovery/page_redo.h | 1 - src/include/access/twophase_rmgr.h | 1 - src/include/access/ustore/knl_uvisibility.h | 26 ++ src/include/access/xact.h | 2 +- src/include/access/xlogproc.h | 10 +- .../knl/knl_guc/knl_instance_attr_storage.h | 3 +- .../knl/knl_guc/knl_session_attr_storage.h | 4 + src/include/knl/knl_instance.h | 8 +- src/include/knl/knl_thread.h | 1 + src/include/miscadmin.h | 1 + src/include/storage/lock/lock.h | 8 +- src/include/storage/proc.h | 11 - src/include/storage/procarray.h | 5 +- src/include/storage/standby.h | 12 +- src/include/utils/guc.h | 2 +- src/test/ha/standby_env.sh | 25 +- .../single_standby_read_base.sh | 28 +- 82 files changed, 1890 insertions(+), 936 deletions(-) create mode 100644 src/gausskernel/storage/access/redo/standby_read/standby_read_proc.cpp diff --git a/src/bin/gs_guc/cluster_guc.conf b/src/bin/gs_guc/cluster_guc.conf index bc9ab6a2a..aeb8f93e6 100755 --- a/src/bin/gs_guc/cluster_guc.conf +++ b/src/bin/gs_guc/cluster_guc.conf @@ -685,6 +685,7 @@ upgrade_mode|int|0,2147483647|NULL|NULL| advance_xlog_file_num|int|0,1000000|NULL|NULL| numa_distribute_mode|string|0,0|NULL|NULL| defer_csn_cleanup_time|int|0,2147483647|ms|NULL| +exrto_standby_read_opt|bool|0,0|NULL|NULL| force_promote|int|0,1|NULL|NULL| max_keep_log_seg|int|0,2147483647|NULL|NULL| datanode_heartbeat_interval|int|1000,60000|ms|The value is best configured less than half of the wal_receiver_timeout and wal_sender_timeout.| diff --git a/src/common/backend/catalog/storage.cpp b/src/common/backend/catalog/storage.cpp index d27c6aa14..922cbda6b 100644 --- a/src/common/backend/catalog/storage.cpp +++ b/src/common/backend/catalog/storage.cpp @@ -1245,7 +1245,7 @@ void smgr_redo_create(RelFileNode rnode, ForkNumber forkNum, char *data) } } -void smgr_redo_truncate_cancel_conflicting_proc(TransactionId latest_removed_xid) +void smgr_redo_truncate_cancel_conflicting_proc(TransactionId latest_removed_xid, XLogRecPtr lsn) { if (IS_EXRTO_READ) { const int max_check_times = 1000; @@ -1256,7 +1256,7 @@ void smgr_redo_truncate_cancel_conflicting_proc(TransactionId latest_removed_xid RedoInterruptCallBack(); check_times++; reach_max_check_times = (check_times == max_check_times); - conflict = proc_array_cancel_conflicting_proc(latest_removed_xid, reach_max_check_times); + conflict = proc_array_cancel_conflicting_proc(latest_removed_xid, lsn, reach_max_check_times); } } } @@ -1264,7 +1264,7 @@ void smgr_redo_truncate_cancel_conflicting_proc(TransactionId latest_removed_xid void xlog_block_smgr_redo_truncate(RelFileNode rnode, BlockNumber blkno, XLogRecPtr lsn, TransactionId latest_removed_xid) { - smgr_redo_truncate_cancel_conflicting_proc(latest_removed_xid); + smgr_redo_truncate_cancel_conflicting_proc(latest_removed_xid, lsn); SMgrRelation reln = smgropen(rnode, InvalidBackendId); smgrcreate(reln, MAIN_FORKNUM, true); UpdateMinRecoveryPoint(lsn, false); diff --git a/src/common/backend/utils/errcodes.txt b/src/common/backend/utils/errcodes.txt index 8e40cec46..c27497612 100644 --- a/src/common/backend/utils/errcodes.txt +++ b/src/common/backend/utils/errcodes.txt @@ -630,3 +630,6 @@ Section: Class SE - Security Error 42714 E ERRCODE_DUPLICATE_POLICY duplicate_policy 42715 E ERRCODE_DUPLICATE_LABEL duplicate_label SE001 E ERRCODE_INVALID_AUDIT_LOG invalid_audit_log + +Section: Class SR - Uncorrected Error & warning +SR001 E ERRCODE_SR_RECOVERY_CONFLICT recovery_conflict \ No newline at end of file diff --git a/src/common/backend/utils/init/globals.cpp b/src/common/backend/utils/init/globals.cpp index 45c289ada..d51879227 100644 --- a/src/common/backend/utils/init/globals.cpp +++ b/src/common/backend/utils/init/globals.cpp @@ -75,12 +75,13 @@ bool will_shutdown = false; * NEXT | 92899 | ? | ? * ********************************************/ -const uint32 GRAND_VERSION_NUM = 92912; +const uint32 GRAND_VERSION_NUM = 92913; /******************************************** * 2.VERSION NUM FOR EACH FEATURE * Please write indescending order. ********************************************/ +const uint32 PARTITION_ACCESS_EXCLUSIVE_LOCK_UPGRADE_VERSION = 92913; const uint32 PAGE_DIST_VERSION_NUM = 92912; const uint32 NODE_REFORM_INFO_VERSION_NUM = 92911; const uint32 GB18030_2022_VERSION_NUM = 92908; diff --git a/src/common/backend/utils/misc/guc.cpp b/src/common/backend/utils/misc/guc.cpp index 8c382b376..add4504fd 100755 --- a/src/common/backend/utils/misc/guc.cpp +++ b/src/common/backend/utils/misc/guc.cpp @@ -6668,14 +6668,15 @@ bool parse_int(const char* value, int* result, int flags, const char** hintmsg) /* * Try to parse value as an 64-bit integer. The accepted format is - * decimal number. + * decimal number, octal, or hexadecimal formats, optionally followed by + * a unit name if "flags" indicates a unit is allowed. * * If the string parses okay, return true, else false. * If okay and result is not NULL, return the value in *result. * If not okay and hintmsg is not NULL, *hintmsg is set to a suitable * HINT message, or NULL if no hint provided. */ -bool parse_int64(const char* value, int64* result, const char** hintmsg) +bool parse_int64(const char *value, int64 *result, int flags, const char **hintmsg) { int64 val; char* endptr = NULL; @@ -6698,7 +6699,7 @@ bool parse_int64(const char* value, int64* result, const char** hintmsg) val = strtol(value, &endptr, 10); #endif - if (endptr == value || *endptr != '\0') { + if (endptr == value) { return false; /* no HINT for integer syntax error */ } @@ -6709,6 +6710,38 @@ bool parse_int64(const char* value, int64* result, const char** hintmsg) return false; } + /* allow whitespace between integer and unit */ + while (isspace((unsigned char)*endptr)) + endptr++; + + /* Handle possible unit conversion before check integer overflow */ + if (*endptr != '\0') { + /* + * Note: the multiple-switch coding technique here is a bit tedious, + * but seems necessary to avoid intermediate-value overflows. + */ + if (flags & GUC_UNIT_MEMORY) { + val = (int64)MemoryUnitConvert(&endptr, val, flags, hintmsg); + } else if (flags & GUC_UNIT_TIME) { + val = (int64)TimeUnitConvert(&endptr, val, flags, hintmsg); + } + + /* allow whitespace after unit */ + while (isspace((unsigned char)*endptr)) + endptr++; + + if (*endptr != '\0') + return false; /* appropriate hint, if any, already set */ + } + + /* Check for integer overflow */ + if (val != (int64)val) { + if (hintmsg != nullptr) + *hintmsg = gettext_noop("Value exceeds integer range."); + + return false; + } + if (result != NULL) { *result = val; } @@ -7266,7 +7299,7 @@ static bool validate_conf_int64(struct config_generic *record, const char *name, int64* newval = (newvalue == NULL ? &tmpnewval : (int64*)newvalue); const char* hintmsg = NULL; - if (!parse_int64(value, newval, &hintmsg)) { + if (!parse_int64(value, newval, conf->gen.flags, &hintmsg)) { ereport(elevel, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("parameter \"%s\" requires a numeric value", name))); diff --git a/src/common/backend/utils/misc/guc/guc_storage.cpp b/src/common/backend/utils/misc/guc/guc_storage.cpp index 1ee3c7c7f..20d996857 100755 --- a/src/common/backend/utils/misc/guc/guc_storage.cpp +++ b/src/common/backend/utils/misc/guc/guc_storage.cpp @@ -1227,7 +1227,6 @@ static void InitStorageConfigureNamesBool() NULL, NULL, NULL}, - {{"enable_time_report", PGC_POSTMASTER, NODE_SINGLENODE, @@ -1251,6 +1250,17 @@ static void InitStorageConfigureNamesBool() NULL, NULL, NULL}, + {{"exrto_standby_read_opt", + PGC_POSTMASTER, + NODE_ALL, + REPLICATION_STANDBY, + gettext_noop("Enable performance optimization of extreme-rto standby read."), + NULL}, + &g_instance.attr.attr_storage.enable_exrto_standby_read_opt, + true, + NULL, + NULL, + NULL}, /* End-of-list marker */ {{NULL, @@ -4101,28 +4111,30 @@ static void InitStorageConfigureNamesInt64() NULL}, #ifndef ENABLE_LITE_MODE {{"max_standby_base_page_size", - PGC_POSTMASTER, + PGC_SIGHUP, NODE_ALL, RESOURCES_RECOVERY, gettext_noop("Sets the max size of base page files on standby"), - NULL}, - &g_instance.attr.attr_storage.max_standby_base_page_size, - INT64CONST(0x4000000000), /* 256GB */ - INT64CONST(0x40000000), /* 1GB */ - INT64CONST(0x7FFFFFFFFFFFFFF), + NULL, + GUC_UNIT_KB}, + &u_sess->attr.attr_storage.max_standby_base_page_size, + 268435456, /* 256GB */ + 1048576, /* 1GB */ + 562949953421311, NULL, NULL, NULL}, {{"max_standby_lsn_info_size", - PGC_POSTMASTER, + PGC_SIGHUP, NODE_ALL, RESOURCES_RECOVERY, gettext_noop("Sets the max size of lsn info files on standby"), - NULL}, - &g_instance.attr.attr_storage.max_standby_lsn_info_size, - INT64CONST(0x4000000000), /* 256GB */ - INT64CONST(0x40000000), /* 1GB */ - INT64CONST(0x7FFFFFFFFFFFFFF), + NULL, + GUC_UNIT_KB}, + &u_sess->attr.attr_storage.max_standby_lsn_info_size, + 268435456, /* 256GB */ + 1048576, /* 1GB */ + 562949953421311, NULL, NULL, NULL}, diff --git a/src/common/backend/utils/time/snapmgr.cpp b/src/common/backend/utils/time/snapmgr.cpp index 68edd053d..d771423e3 100644 --- a/src/common/backend/utils/time/snapmgr.cpp +++ b/src/common/backend/utils/time/snapmgr.cpp @@ -583,6 +583,7 @@ void StreamTxnContextSetSnapShot(void* snapshotPtr) u_sess->utils_cxt.CurrentSnapshot->xmax = snapshot->xmax; u_sess->utils_cxt.CurrentSnapshot->timeline = snapshot->timeline; u_sess->utils_cxt.CurrentSnapshot->snapshotcsn = snapshot->snapshotcsn; + u_sess->utils_cxt.CurrentSnapshot->read_lsn = snapshot->read_lsn; u_sess->utils_cxt.CurrentSnapshot->curcid = snapshot->curcid; diff --git a/src/gausskernel/optimizer/commands/tablespace.cpp b/src/gausskernel/optimizer/commands/tablespace.cpp index dd36b8c32..3ba6410f8 100644 --- a/src/gausskernel/optimizer/commands/tablespace.cpp +++ b/src/gausskernel/optimizer/commands/tablespace.cpp @@ -97,7 +97,7 @@ #include "storage/tcap.h" static void create_tablespace_directories(const char* location, const Oid tablespaceoid); -static bool destroy_tablespace_directories(Oid tablespaceoid, bool redo); +static bool destroy_tablespace_directories(Oid tablespaceoid, bool redo, bool is_exrto_read = false); static void createtbspc_abort_callback(bool isCommit, const void* arg); Datum CanonicalizeTablespaceOptions(Datum datum); @@ -1487,7 +1487,7 @@ static void createtbspc_abort_callback(bool isCommit, const void* arg) * * Returns TRUE if successful, FALSE if some subdirectory is not empty */ -static bool destroy_tablespace_directories(Oid tablespaceoid, bool redo) +static bool destroy_tablespace_directories(Oid tablespaceoid, bool redo, bool is_exrto_read) { char* linkloc = NULL; char* linkloc_with_version_dir = NULL; @@ -1607,7 +1607,9 @@ static bool destroy_tablespace_directories(Oid tablespaceoid, bool redo) if (rmdir(subfile) < 0) ereport(redo ? LOG : ERROR, (errcode_for_file_access(), errmsg("could not remove directory \"%s\": %m", subfile))); - + if (is_exrto_read) { + rmtree(subfile, true); + } if (spc) { spc_unlock(spc); } @@ -2634,7 +2636,7 @@ void xlog_drop_tblspc(Oid tsId) * that would crash the database and require manual intervention * before we could get past this WAL record on restart). */ - if (!destroy_tablespace_directories(tsId, true)) + if (!destroy_tablespace_directories(tsId, true, true)) ereport(LOG, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("directories for tablespace %u could not be removed", tsId), diff --git a/src/gausskernel/process/postmaster/postmaster.cpp b/src/gausskernel/process/postmaster/postmaster.cpp index 2a4e68afb..8c517e0a2 100644 --- a/src/gausskernel/process/postmaster/postmaster.cpp +++ b/src/gausskernel/process/postmaster/postmaster.cpp @@ -4329,7 +4329,7 @@ static int ServerLoop(void) if (g_instance.attr.attr_storage.enable_ustore && g_instance.pid_cxt.UndoRecyclerPID == 0 && - pmState == PM_RUN) { + (pmState == PM_RUN || IS_EXRTO_STANDBY_READ)) { g_instance.pid_cxt.UndoRecyclerPID = initialize_util_thread(UNDO_RECYCLER); } @@ -6921,6 +6921,9 @@ static void reaper(SIGNAL_ARGS) g_instance.fatal_error = false; g_instance.demotion = NoDemote; t_thrd.postmaster_cxt.ReachedNormalRunning = true; + if ((IS_EXRTO_STANDBY_READ) && (g_instance.pid_cxt.UndoRecyclerPID!= 0)) { + signal_child(g_instance.pid_cxt.UndoRecyclerPID, SIGTERM); + } pmState = PM_RUN; if (t_thrd.postmaster_cxt.HaShmData && (t_thrd.postmaster_cxt.HaShmData->current_mode == STANDBY_MODE || diff --git a/src/gausskernel/process/stream/streamMain.cpp b/src/gausskernel/process/stream/streamMain.cpp index 9cc8c2be5..b4206856f 100755 --- a/src/gausskernel/process/stream/streamMain.cpp +++ b/src/gausskernel/process/stream/streamMain.cpp @@ -486,23 +486,19 @@ static void execute_stream_plan(StreamProducer* producer) PortalDefineQuery(portal, NULL, "DUMMY", commandTag, lappend(NULL, planstmt), NULL); - /* - * Start the portal. No parameters here. - */ - PortalStart(portal, producer->getParams(), 0, producer->getSnapShot()); - /* The value of snapshot.read_lsn may be assigned to thread A and used on thread B. So we should reassigned read_lsn to t_thrd of thread B */ if (unlikely(IS_EXRTO_STANDBY_READ && producer->getSnapShot() != NULL)) { t_thrd.proc->exrto_read_lsn = producer->getSnapShot()->read_lsn; t_thrd.proc->exrto_min = t_thrd.proc->exrto_read_lsn; + reset_invalidation_cache(); } - /* The value of snapshot.read_lsn may be assigned to thread A and used on thread B. - So we should reassigned read_lsn to t_thrd of thread B */ - if (unlikely(IS_EXRTO_STANDBY_READ && producer->getSnapShot() != NULL)) { - t_thrd.proc->exrto_read_lsn = producer->getSnapShot()->read_lsn; - } + /* + * Start the portal. No parameters here. + */ + PortalStart(portal, producer->getParams(), 0, producer->getSnapShot()); + format = 0; PortalSetResultFormat(portal, 1, &format); diff --git a/src/gausskernel/process/tcop/postgres.cpp b/src/gausskernel/process/tcop/postgres.cpp index ced83182d..74e69a81c 100755 --- a/src/gausskernel/process/tcop/postgres.cpp +++ b/src/gausskernel/process/tcop/postgres.cpp @@ -6717,7 +6717,7 @@ void ProcessInterrupts(void) " database and repeat your command."))); else ereport(ERROR, - (errcode(ERRCODE_T_R_SERIALIZATION_FAILURE), + (errcode(ERRCODE_SR_RECOVERY_CONFLICT), errmsg("canceling statement due to conflict with recovery"), errdetail_recovery_conflict())); } diff --git a/src/gausskernel/process/threadpool/knl_instance.cpp b/src/gausskernel/process/threadpool/knl_instance.cpp index 518e71f7f..9b553aaeb 100755 --- a/src/gausskernel/process/threadpool/knl_instance.cpp +++ b/src/gausskernel/process/threadpool/knl_instance.cpp @@ -329,6 +329,7 @@ static void knl_g_parallel_redo_init(knl_g_parallel_redo_context* predo_cxt) rc = memset_s(&predo_cxt->redoCpuBindcontrl, sizeof(RedoCpuBindControl), 0, sizeof(RedoCpuBindControl)); securec_check(rc, "", ""); predo_cxt->global_recycle_lsn = InvalidXLogRecPtr; + predo_cxt->exrto_recyle_xmin = 0; predo_cxt->exrto_snapshot = (ExrtoSnapshot)MemoryContextAllocZero( INSTANCE_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_STORAGE), sizeof(ExrtoSnapshotData)); predo_cxt->redoItemHash = NULL; @@ -337,6 +338,9 @@ static void knl_g_parallel_redo_init(knl_g_parallel_redo_context* predo_cxt) predo_cxt->standby_read_delay_ddl_stat.next_index_can_insert = 0; predo_cxt->standby_read_delay_ddl_stat.next_index_need_unlink = 0; predo_cxt->max_clog_pageno = 0; + predo_cxt->buffer_pin_wait_buf_ids = NULL; + predo_cxt->buffer_pin_wait_buf_len = 0; + predo_cxt->exrto_send_lsn_forworder_time = 0; } static void knl_g_parallel_decode_init(knl_g_parallel_decode_context* pdecode_cxt) @@ -512,6 +516,7 @@ static void KnlGUndoInit(knl_g_undo_context *undoCxt) undoCxt->undoChainTotalSize = 0; undoCxt->globalFrozenXid = InvalidTransactionId; undoCxt->globalRecycleXid = InvalidTransactionId; + undoCxt->hotStandbyRecycleXid = InvalidTransactionId; undoCxt->is_exrto_residual_undo_file_recycled = false; } diff --git a/src/gausskernel/process/threadpool/knl_thread.cpp b/src/gausskernel/process/threadpool/knl_thread.cpp index 0f46ecc62..8ef2161e5 100755 --- a/src/gausskernel/process/threadpool/knl_thread.cpp +++ b/src/gausskernel/process/threadpool/knl_thread.cpp @@ -1131,6 +1131,7 @@ static void KnlTUndorecyclerInit(knl_t_undorecycler_context* undorecyclerCxt) { undorecyclerCxt->got_SIGHUP = false; undorecyclerCxt->shutdown_requested = false; + undorecyclerCxt->is_recovery_in_progress = false; } static void KnlTUstoreInit(knl_u_ustore_context *ustoreCxt) diff --git a/src/gausskernel/storage/access/common/reloptions.cpp b/src/gausskernel/storage/access/common/reloptions.cpp index 28163d71e..afdcf9690 100644 --- a/src/gausskernel/storage/access/common/reloptions.cpp +++ b/src/gausskernel/storage/access/common/reloptions.cpp @@ -1232,7 +1232,7 @@ static void parse_one_reloption(relopt_value *option, const char *text_str, int case RELOPT_TYPE_INT64: { relopt_int64 *optint = (relopt_int64 *)option->gen; - parsed = parse_int64(value, &option->values.int64_val, NULL); + parsed = parse_int64(value, &option->values.int64_val, 0, NULL); if (validate && !parsed) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), diff --git a/src/gausskernel/storage/access/heap/heapam.cpp b/src/gausskernel/storage/access/heap/heapam.cpp index 5625c0d60..05a5338b8 100755 --- a/src/gausskernel/storage/access/heap/heapam.cpp +++ b/src/gausskernel/storage/access/heap/heapam.cpp @@ -8741,7 +8741,7 @@ static void heap_xlog_cleanup_info(XLogReaderState* record) return; } - if (InHotStandby && g_supportHotStandby && !IS_EXRTO_READ) { + if (InHotStandby && g_supportHotStandby) { XLogRecPtr lsn = record->EndRecPtr; ResolveRecoveryConflictWithSnapshot(xlrec->latestRemovedXid, tmp_node, lsn); } diff --git a/src/gausskernel/storage/access/nbtree/nbtpage.cpp b/src/gausskernel/storage/access/nbtree/nbtpage.cpp index 8173ccb92..33fae0154 100644 --- a/src/gausskernel/storage/access/nbtree/nbtpage.cpp +++ b/src/gausskernel/storage/access/nbtree/nbtpage.cpp @@ -24,6 +24,7 @@ #include "postgres.h" #include "knl/knl_variable.h" +#include "access/extreme_rto/standby_read/standby_read_base.h" #include "access/hio.h" #include "access/multi_redo_api.h" #include "access/nbtree.h" @@ -507,10 +508,21 @@ void _bt_checkbuffer_valid(Relation rel, Buffer buf) } } +void exrto_dump_btree_info(Relation rel, BlockNumber blkno, BlockNumber par_blkno) +{ + if (RelationIsUstoreIndex(rel)) { + extreme_rto_standby_read::dump_error_all_info(rel->rd_node, 0, blkno); + extreme_rto_standby_read::dump_error_all_info(rel->rd_node, 0, par_blkno); + } else { + extreme_rto_standby_read::dump_error_all_info(rel->rd_node, 0, blkno); + extreme_rto_standby_read::dump_error_all_info(rel->rd_node, 0, par_blkno); + } +} + /* * _bt_checkpage() -- Verify that a freshly-read page looks sane. */ -void _bt_checkpage(Relation rel, Buffer buf) +void _bt_checkpage(Relation rel, Buffer buf, BlockNumber par_blkno) { Page page = BufferGetPage(buf); /* @@ -519,11 +531,16 @@ void _bt_checkpage(Relation rel, Buffer buf) * page header or is all-zero. We have to defend against the all-zero * case, however. */ - if (PageIsNew(page)) - ereport(ERROR, (errcode(ERRCODE_INDEX_CORRUPTED), - errmsg("index \"%s\" contains unexpected zero page at block %u", RelationGetRelationName(rel), - BufferGetBlockNumber(buf)), - errhint("Please REINDEX it."))); + if (PageIsNew(page)) { + PageHeader phdr = (PageHeader)page; + exrto_dump_btree_info(rel, BufferGetBlockNumber(buf), par_blkno); + ereport(ERROR, + (errcode(ERRCODE_INDEX_CORRUPTED), + errmsg("index \"%s\" oid: %u contains unexpected zero page at block %u, pd_upper %d pd_lower %d", + RelationGetRelationName(rel), rel->rd_id, BufferGetBlockNumber(buf), phdr->pd_upper, + phdr->pd_lower), + errhint("Please REINDEX it."))); + } /* * Additionally check that the special area looks sane. @@ -714,7 +731,7 @@ Buffer _bt_getbuf(Relation rel, BlockNumber blkno, int access) * is when the target page is the same one already in the buffer. */ FORCE_INLINE -Buffer _bt_relandgetbuf(Relation rel, Buffer obuf, BlockNumber blkno, int access) +Buffer _bt_relandgetbuf(Relation rel, Buffer obuf, BlockNumber blkno, int access, BlockNumber par_blkno) { Buffer buf; @@ -724,7 +741,7 @@ Buffer _bt_relandgetbuf(Relation rel, Buffer obuf, BlockNumber blkno, int access buf = ReleaseAndReadBuffer(obuf, rel, blkno); _bt_checkbuffer_valid(rel, buf); LockBuffer(buf, access); - _bt_checkpage(rel, buf); + _bt_checkpage(rel, buf, par_blkno); return buf; } diff --git a/src/gausskernel/storage/access/redo/redo_ginxlog.cpp b/src/gausskernel/storage/access/redo/redo_ginxlog.cpp index 34557f42e..2ac61e357 100644 --- a/src/gausskernel/storage/access/redo/redo_ginxlog.cpp +++ b/src/gausskernel/storage/access/redo/redo_ginxlog.cpp @@ -126,7 +126,7 @@ static XLogRecParseState *GinXlogSplitParseBlock(XLogReaderState *record, uint32 { XLogRecParseState *recordstatehead = NULL; XLogParseBufferAllocListFunc(record, &recordstatehead, NULL); - XLogRecSetBlockDataState(record, GIN_SPLIT_LEFT_BLOCK_NUM, recordstatehead); + XLogRecSetBlockDataState(record, GIN_SPLIT_LEFT_BLOCK_NUM, recordstatehead, BLOCK_DATA_MAIN_DATA_TYPE, true); XLogRecParseState *blockstate = NULL; XLogParseBufferAllocListFunc(record, &blockstate, recordstatehead); @@ -140,7 +140,7 @@ static XLogRecParseState *GinXlogSplitParseBlock(XLogReaderState *record, uint32 if (isRoot) { XLogParseBufferAllocListFunc(record, &blockstate, recordstatehead); - XLogRecSetBlockDataState(record, GIN_SPLIT_ROOT_BLOCK_NUM, blockstate); + XLogRecSetBlockDataState(record, GIN_SPLIT_ROOT_BLOCK_NUM, blockstate, BLOCK_DATA_MAIN_DATA_TYPE, true); ++(*blocknum); } @@ -177,14 +177,14 @@ static XLogRecParseState *GinXlogDeleteParseBlock(XLogReaderState *record, uint3 { XLogRecParseState *recordstatehead = NULL; XLogParseBufferAllocListFunc(record, &recordstatehead, NULL); - XLogRecSetBlockDataState(record, GIN_DELETE_D_PAGE_BLOCK_NUM, recordstatehead); + XLogRecSetBlockDataState(record, GIN_DELETE_D_PAGE_BLOCK_NUM, recordstatehead, BLOCK_DATA_MAIN_DATA_TYPE, true); XLogRecParseState *blockstate = NULL; XLogParseBufferAllocListFunc(record, &blockstate, recordstatehead); - XLogRecSetBlockDataState(record, GIN_DELETE_P_PAGE_BLOCK_NUM, blockstate); + XLogRecSetBlockDataState(record, GIN_DELETE_P_PAGE_BLOCK_NUM, blockstate, BLOCK_DATA_MAIN_DATA_TYPE, true); XLogParseBufferAllocListFunc(record, &blockstate, recordstatehead); - XLogRecSetBlockDataState(record, GIN_DELETE_L_PAGE_BLOCK_NUM, blockstate); + XLogRecSetBlockDataState(record, GIN_DELETE_L_PAGE_BLOCK_NUM, blockstate, BLOCK_DATA_MAIN_DATA_TYPE, true); *blocknum = 3; return recordstatehead; @@ -194,7 +194,7 @@ static XLogRecParseState *GinXlogUpdateMetaPageParseBlock(XLogReaderState *recor { XLogRecParseState *recordstatehead = NULL; XLogParseBufferAllocListFunc(record, &recordstatehead, NULL); - XLogRecSetBlockDataState(record, GIN_META_PAGE_BLOCK_NUM, recordstatehead); + XLogRecSetBlockDataState(record, GIN_META_PAGE_BLOCK_NUM, recordstatehead, BLOCK_DATA_MAIN_DATA_TYPE, true); *blocknum = 1; ginxlogUpdateMeta *data = (ginxlogUpdateMeta *)XLogRecGetData(record); @@ -221,7 +221,8 @@ static XLogRecParseState *GinXlogDeleteListPageParseBlock(XLogReaderState *recor { XLogRecParseState *recordstatehead = NULL; XLogParseBufferAllocListFunc(record, &recordstatehead, NULL); - XLogRecSetBlockDataState(record, GIN_DELETE_LIST_META_PAGE_BLOCK_NUM, recordstatehead); + XLogRecSetBlockDataState(record, GIN_DELETE_LIST_META_PAGE_BLOCK_NUM, recordstatehead, BLOCK_DATA_MAIN_DATA_TYPE, + true); *blocknum = 1; ginxlogDeleteListPages *data = (ginxlogDeleteListPages *)XLogRecGetData(record); @@ -229,7 +230,7 @@ static XLogRecParseState *GinXlogDeleteListPageParseBlock(XLogReaderState *recor for (int32 i = 0; i < data->ndeleted; i++) { XLogRecParseState *blockstate = NULL; XLogParseBufferAllocListFunc(record, &blockstate, recordstatehead); - XLogRecSetBlockDataState(record, i + 1, blockstate); + XLogRecSetBlockDataState(record, i + 1, blockstate, BLOCK_DATA_MAIN_DATA_TYPE, true); ++(*blocknum); } diff --git a/src/gausskernel/storage/access/redo/redo_gistxlog.cpp b/src/gausskernel/storage/access/redo/redo_gistxlog.cpp index c859e8267..4f0cbf0f2 100644 --- a/src/gausskernel/storage/access/redo/redo_gistxlog.cpp +++ b/src/gausskernel/storage/access/redo/redo_gistxlog.cpp @@ -202,11 +202,13 @@ static XLogRecParseState *GistXlogPageSplitParseBlock(XLogReaderState *record, u BlockNumber blkno; XLogRecGetBlockTag(record, i + 1, NULL, NULL, &blkno); if (blkno == GIST_ROOT_BLKNO) { + XLogRecSetBlockDataState(record, i + 1, blockstate, BLOCK_DATA_MAIN_DATA_TYPE, true); XLogRecSetAuxiBlkNumState(&blockstate->blockparse.extra_rec.blockdatarec, InvalidForkNumber, InvalidForkNumber); isrootsplit = true; } if (blkno != GIST_ROOT_BLKNO) { + XLogRecSetBlockDataState(record, i + 1, blockstate); uint32 flag; if ((i < xldata->npage - 1) && !isrootsplit && xldata->markfollowright) flag = 1; diff --git a/src/gausskernel/storage/access/redo/redo_heapam.cpp b/src/gausskernel/storage/access/redo/redo_heapam.cpp index ff01c545f..28ce6f532 100755 --- a/src/gausskernel/storage/access/redo/redo_heapam.cpp +++ b/src/gausskernel/storage/access/redo/redo_heapam.cpp @@ -991,7 +991,7 @@ static XLogRecParseState *HeapXlogBaseShiftParseBlock(XLogReaderState *record, u if (recordstatehead == NULL) { return NULL; } - XLogRecSetBlockDataState(record, HEAP_BASESHIFT_ORIG_BLOCK_NUM, recordstatehead); + XLogRecSetBlockDataState(record, HEAP_BASESHIFT_ORIG_BLOCK_NUM, recordstatehead, BLOCK_DATA_MAIN_DATA_TYPE, true); return recordstatehead; } @@ -1018,7 +1018,7 @@ static XLogRecParseState *HeapXlogLockParseBlock(XLogReaderState *record, uint32 if (recordstatehead == NULL) { return NULL; } - XLogRecSetBlockDataState(record, HEAP_LOCK_ORIG_BLOCK_NUM, recordstatehead); + XLogRecSetBlockDataState(record, HEAP_LOCK_ORIG_BLOCK_NUM, recordstatehead, BLOCK_DATA_MAIN_DATA_TYPE, true); return recordstatehead; } @@ -1080,37 +1080,13 @@ XLogRecParseState *HeapRedoParseToBlock(XLogReaderState *record, uint32 *blocknu static XLogRecParseState *HeapXlogFreezeParseBlock(XLogReaderState *record, uint32 *blocknum) { XLogRecParseState *recordstatehead = NULL; - XLogRecParseState *blockstate = NULL; *blocknum = 1; XLogParseBufferAllocListFunc(record, &recordstatehead, NULL); if (recordstatehead == NULL) { return NULL; } - XLogRecSetBlockDataState(record, HEAP_FREEZE_ORIG_BLOCK_NUM, recordstatehead); - - /* - * In Hot Standby mode, ensure that there's no queries running which still - * consider the frozen xids as running. - */ - if (g_supportHotStandby) { - (*blocknum)++; - /* need notify hot standby */ - XLogParseBufferAllocListFunc(record, &blockstate, recordstatehead); - if (blockstate == NULL) { - return NULL; - } - RelFileNode rnode; - xl_heap_freeze *xlrec = (xl_heap_freeze *)XLogRecGetData(record); - TransactionId cutoff_xid = xlrec->cutoff_xid; - - XLogRecGetBlockTag(record, HEAP_FREEZE_ORIG_BLOCK_NUM, &rnode, NULL, NULL); - - RelFileNodeForkNum filenode = - RelFileNodeForkNumFill(&rnode, InvalidBackendId, InvalidForkNumber, InvalidBlockNumber); - XLogRecSetBlockCommonState(record, BLOCK_DATA_INVALIDMSG_TYPE, filenode, blockstate); - XLogRecSetInvalidMsgState(&blockstate->blockparse.extra_rec.blockinvalidmsg, cutoff_xid); - } + XLogRecSetBlockDataState(record, HEAP_FREEZE_ORIG_BLOCK_NUM, recordstatehead, BLOCK_DATA_MAIN_DATA_TYPE, true); return recordstatehead; } @@ -1118,105 +1094,52 @@ static XLogRecParseState *HeapXlogFreezeParseBlock(XLogReaderState *record, uint static XLogRecParseState *HeapXlogInvalidParseBlock(XLogReaderState *record, uint32 *blocknum) { *blocknum = 1; - XLogRecParseState *blockstate = NULL; XLogRecParseState *recordstatehead = NULL; XLogParseBufferAllocListFunc(record, &recordstatehead, NULL); if (recordstatehead == NULL) { return NULL; } - XLogRecSetBlockDataState(record, HEAP_FREEZE_ORIG_BLOCK_NUM, recordstatehead); - - /* - * In Hot Standby mode, ensure that there's no queries running which still consider the - * invalid xids as running. - */ - if (g_supportHotStandby) { - (*blocknum)++; - /* need notify hot standby */ - XLogParseBufferAllocListFunc(record, &blockstate, recordstatehead); - if (blockstate == NULL) { - return NULL; - } - /* get cutoff xid */ - xl_heap_invalid *xlrecInvalid = (xl_heap_invalid *)XLogRecGetData(record); - TransactionId cutoff_xid = xlrecInvalid->cutoff_xid; - RelFileNode rnode; - - XLogRecGetBlockTag(record, HEAP_FREEZE_ORIG_BLOCK_NUM, &rnode, NULL, NULL); - - RelFileNodeForkNum filenode = - RelFileNodeForkNumFill(&rnode, InvalidBackendId, InvalidForkNumber, InvalidBlockNumber); - XLogRecSetBlockCommonState(record, BLOCK_DATA_INVALIDMSG_TYPE, filenode, blockstate); - XLogRecSetInvalidMsgState(&blockstate->blockparse.extra_rec.blockinvalidmsg, cutoff_xid); - } + XLogRecSetBlockDataState(record, HEAP_FREEZE_ORIG_BLOCK_NUM, recordstatehead, BLOCK_DATA_MAIN_DATA_TYPE, true); return recordstatehead; } static XLogRecParseState *HeapXlogCleanParseBlock(XLogReaderState *record, uint32 *blocknum) { - xl_heap_clean *xlrec = (xl_heap_clean *)XLogRecGetData(record); XLogRecParseState *recordstatehead = NULL; - XLogRecParseState *blockstate = NULL; *blocknum = 1; XLogParseBufferAllocListFunc(record, &recordstatehead, NULL); if (recordstatehead == NULL) { return NULL; } - XLogRecSetBlockDataState(record, HEAP_CLEAN_ORIG_BLOCK_NUM, recordstatehead); + XLogRecSetBlockDataState(record, HEAP_CLEAN_ORIG_BLOCK_NUM, recordstatehead, BLOCK_DATA_MAIN_DATA_TYPE, true); - /* - * We're about to remove tuples. In Hot Standby mode, ensure that there's - * no queries running for which the removed tuples are still visible. - * - * Not all HEAP2_CLEAN records remove tuples with xids, so we only want to - * conflict on the records that cause MVCC failures for user queries. If - * latestRemovedXid is invalid, skip conflict processing. - */ - if (g_supportHotStandby && TransactionIdIsValid(xlrec->latestRemovedXid)) { - (*blocknum)++; - /* need notify hot standby */ - XLogParseBufferAllocListFunc(record, &blockstate, recordstatehead); - if (blockstate == NULL) { - return NULL; - } - RelFileNode rnode; - XLogRecGetBlockTag(record, HEAP_CLEAN_ORIG_BLOCK_NUM, &rnode, NULL, NULL); - - RelFileNodeForkNum filenode = - RelFileNodeForkNumFill(&rnode, InvalidBackendId, InvalidForkNumber, InvalidBlockNumber); - XLogRecSetBlockCommonState(record, BLOCK_DATA_INVALIDMSG_TYPE, filenode, blockstate); - XLogRecSetInvalidMsgState(&blockstate->blockparse.extra_rec.blockinvalidmsg, xlrec->latestRemovedXid); - } return recordstatehead; } static XLogRecParseState *HeapXlogCleanupInfoParseBlock(XLogReaderState *record, uint32 *blocknum) { XLogRecParseState *recordstatehead = NULL; + RelFileNodeOld *rnode = NULL; + ForkNumber forknum = MAIN_FORKNUM; + BlockNumber blkno = InvalidBlockNumber; - /* Backup blocks are not used in cleanup_info records */ - Assert(!XLogRecHasAnyBlockRefs(record)); + (*blocknum)++; + XLogParseBufferAllocListFunc(record, &recordstatehead, NULL); - *blocknum = 0; - if (g_supportHotStandby) { - (*blocknum)++; - XLogParseBufferAllocListFunc(record, &recordstatehead, NULL); - if (recordstatehead == NULL) { - return NULL; - } + xl_heap_cleanup_info *xlrec = (xl_heap_cleanup_info *)XLogRecGetData(record); + rnode = &(xlrec->node); + forknum = MAIN_FORKNUM; + RelFileNode tmp_node; + RelFileNodeCopy(tmp_node, *rnode, (int2)XLogRecGetBucketId(record)); + tmp_node.opt = 0; + RelFileNodeForkNum filenode = RelFileNodeForkNumFill(&tmp_node, InvalidBackendId, forknum, blkno); + XLogRecSetBlockCommonState(record, BLOCK_DATA_CLEANUP_TYPE, filenode, recordstatehead); - xl_heap_cleanup_info *xlrec = (xl_heap_cleanup_info *)XLogRecGetData(record); - RelFileNode rnode; - RelFileNodeCopy(rnode, xlrec->node, XLogRecGetBucketId(record)); + wal_rec_set_clean_up_info_state(&(recordstatehead->blockparse.extra_rec.clean_up_info), xlrec->latestRemovedXid); - RelFileNodeForkNum filenode = - RelFileNodeForkNumFill(&rnode, InvalidBackendId, InvalidForkNumber, InvalidBlockNumber); - XLogRecSetBlockCommonState(record, BLOCK_DATA_INVALIDMSG_TYPE, filenode, recordstatehead); - XLogRecSetInvalidMsgState(&recordstatehead->blockparse.extra_rec.blockinvalidmsg, xlrec->latestRemovedXid); - } return recordstatehead; } @@ -1230,7 +1153,7 @@ static XLogRecParseState *HeapXlogVisibleParseBlock(XLogReaderState *record, uin if (recordstatehead == NULL) { return NULL; } - XLogRecSetBlockDataState(record, HEAP_VISIBLE_VM_BLOCK_NUM, recordstatehead); + XLogRecSetBlockDataState(record, HEAP_VISIBLE_VM_BLOCK_NUM, recordstatehead, BLOCK_DATA_MAIN_DATA_TYPE, true); if (XLogRecHasBlockRef(record, HEAP_VISIBLE_DATA_BLOCK_NUM)) { (*blocknum)++; @@ -1238,26 +1161,9 @@ static XLogRecParseState *HeapXlogVisibleParseBlock(XLogReaderState *record, uin if (blockstate == NULL) { return NULL; } - XLogRecSetBlockDataState(record, HEAP_VISIBLE_DATA_BLOCK_NUM, blockstate); - } - - if (g_supportHotStandby) { - (*blocknum)++; - XLogParseBufferAllocListFunc(record, &blockstate, recordstatehead); - if (blockstate == NULL) { - return NULL; - } - RelFileNode rnode; - xl_heap_visible *xlrec = (xl_heap_visible *)XLogRecGetData(record); - - XLogRecGetBlockTag(record, HEAP_VISIBLE_VM_BLOCK_NUM, &rnode, NULL, NULL); - - RelFileNodeForkNum filenode = - RelFileNodeForkNumFill(&rnode, InvalidBackendId, InvalidForkNumber, InvalidBlockNumber); - XLogRecSetBlockCommonState(record, BLOCK_DATA_INVALIDMSG_TYPE, filenode, blockstate); - XLogRecSetInvalidMsgState(&blockstate->blockparse.extra_rec.blockinvalidmsg, xlrec->cutoff_xid); + XLogRecSetBlockDataState(record, HEAP_VISIBLE_DATA_BLOCK_NUM, blockstate, BLOCK_DATA_MAIN_DATA_TYPE, true); } - + return recordstatehead; } diff --git a/src/gausskernel/storage/access/redo/redo_nbtxlog.cpp b/src/gausskernel/storage/access/redo/redo_nbtxlog.cpp index 9b62eb03e..5213ab411 100644 --- a/src/gausskernel/storage/access/redo/redo_nbtxlog.cpp +++ b/src/gausskernel/storage/access/redo/redo_nbtxlog.cpp @@ -593,9 +593,12 @@ XLogRecParseState *BtreeXlogInsertParseBlock(XLogReaderState *record, uint32 *bl if (recordstatehead == NULL) { return NULL; } - XLogRecSetBlockDataState(record, BTREE_INSERT_ORIG_BLOCK_NUM, recordstatehead); - if (info != XLOG_BTREE_INSERT_LEAF) { + if (info == XLOG_BTREE_INSERT_LEAF) { + XLogRecSetBlockDataState(record, BTREE_INSERT_ORIG_BLOCK_NUM, recordstatehead); + } else { + XLogRecSetBlockDataState(record, BTREE_INSERT_ORIG_BLOCK_NUM, recordstatehead, BLOCK_DATA_MAIN_DATA_TYPE, + true); (*blocknum)++; XLogParseBufferAllocListFunc(record, &blockstate, recordstatehead); if (blockstate == NULL) { @@ -610,7 +613,7 @@ XLogRecParseState *BtreeXlogInsertParseBlock(XLogReaderState *record, uint32 *bl if (blockstate == NULL) { return NULL; } - XLogRecSetBlockDataState(record, BTREE_INSERT_META_BLOCK_NUM, blockstate); + XLogRecSetBlockDataState(record, BTREE_INSERT_META_BLOCK_NUM, blockstate, BLOCK_DATA_MAIN_DATA_TYPE, true); } return recordstatehead; @@ -636,7 +639,7 @@ static XLogRecParseState *BtreeXlogSplitParseBlock(XLogReaderState *record, uint if (recordstatehead == NULL) { return NULL; } - XLogRecSetBlockDataState(record, BTREE_SPLIT_LEFT_BLOCK_NUM, recordstatehead); + XLogRecSetBlockDataState(record, BTREE_SPLIT_LEFT_BLOCK_NUM, recordstatehead, BLOCK_DATA_MAIN_DATA_TYPE, true); XLogRecSetAuxiBlkNumState(&recordstatehead->blockparse.extra_rec.blockdatarec, rightsib, InvalidForkNumber); (*blocknum)++; @@ -644,6 +647,8 @@ static XLogRecParseState *BtreeXlogSplitParseBlock(XLogReaderState *record, uint if (blockstate == NULL) { return NULL; } + + // no need restore base page, because this is a new page XLogRecSetBlockDataState(record, BTREE_SPLIT_RIGHT_BLOCK_NUM, blockstate); XLogRecSetAuxiBlkNumState(&blockstate->blockparse.extra_rec.blockdatarec, rnext, leftsib); @@ -653,7 +658,7 @@ static XLogRecParseState *BtreeXlogSplitParseBlock(XLogReaderState *record, uint if (blockstate == NULL) { return NULL; } - XLogRecSetBlockDataState(record, BTREE_SPLIT_RIGHTNEXT_BLOCK_NUM, blockstate); + XLogRecSetBlockDataState(record, BTREE_SPLIT_RIGHTNEXT_BLOCK_NUM, blockstate, BLOCK_DATA_MAIN_DATA_TYPE, true); XLogRecSetAuxiBlkNumState(&blockstate->blockparse.extra_rec.blockdatarec, rightsib, InvalidForkNumber); } @@ -663,6 +668,8 @@ static XLogRecParseState *BtreeXlogSplitParseBlock(XLogReaderState *record, uint if (blockstate == NULL) { return NULL; } + + // just clear split flag,no need restore base page XLogRecSetBlockDataState(record, BTREE_SPLIT_CHILD_BLOCK_NUM, blockstate); } @@ -679,7 +686,7 @@ static XLogRecParseState *BtreeXlogVacuumParseBlock(XLogReaderState *record, uin return NULL; } - XLogRecSetBlockDataState(record, BTREE_VACUUM_ORIG_BLOCK_NUM, recordstatehead); + XLogRecSetBlockDataState(record, BTREE_VACUUM_ORIG_BLOCK_NUM, recordstatehead, BLOCK_DATA_MAIN_DATA_TYPE, true); return recordstatehead; } @@ -693,7 +700,7 @@ static XLogRecParseState *BtreeXlogDeleteParseBlock(XLogReaderState *record, uin return NULL; } - XLogRecSetBlockDataState(record, BTREE_DELETE_ORIG_BLOCK_NUM, recordstatehead); + XLogRecSetBlockDataState(record, BTREE_DELETE_ORIG_BLOCK_NUM, recordstatehead, BLOCK_DATA_MAIN_DATA_TYPE, true); /* for hot standby, need to reslove the conflict */ { @@ -713,14 +720,14 @@ static XLogRecParseState *BtreeXlogMarkHalfdeadParseBlock(XLogReaderState *recor return NULL; } - XLogRecSetBlockDataState(record, BTREE_HALF_DEAD_PARENT_PAGE_NUM, recordstatehead); + XLogRecSetBlockDataState(record, BTREE_HALF_DEAD_PARENT_PAGE_NUM, recordstatehead, BLOCK_DATA_MAIN_DATA_TYPE, true); (*blocknum)++; XLogParseBufferAllocListFunc(record, &blockstate, recordstatehead); if (blockstate == NULL) { return NULL; } - XLogRecSetBlockDataState(record, BTREE_HALF_DEAD_LEAF_PAGE_NUM, blockstate); + XLogRecSetBlockDataState(record, BTREE_HALF_DEAD_LEAF_PAGE_NUM, blockstate, BLOCK_DATA_MAIN_DATA_TYPE, true); return recordstatehead; } @@ -738,7 +745,7 @@ static XLogRecParseState *BtreeXlogUnlinkPageParseBlock(XLogReaderState *record, return NULL; } - XLogRecSetBlockDataState(record, BTREE_UNLINK_PAGE_RIGHT_NUM, recordstatehead); + XLogRecSetBlockDataState(record, BTREE_UNLINK_PAGE_RIGHT_NUM, recordstatehead, BLOCK_DATA_MAIN_DATA_TYPE, true); if (xlrec->leftsib != P_NONE) { (*blocknum)++; @@ -746,7 +753,7 @@ static XLogRecParseState *BtreeXlogUnlinkPageParseBlock(XLogReaderState *record, if (blockstate == NULL) { return NULL; } - XLogRecSetBlockDataState(record, BTREE_UNLINK_PAGE_LEFT_NUM, blockstate); + XLogRecSetBlockDataState(record, BTREE_UNLINK_PAGE_LEFT_NUM, blockstate, BLOCK_DATA_MAIN_DATA_TYPE, true); } (*blocknum)++; @@ -754,7 +761,7 @@ static XLogRecParseState *BtreeXlogUnlinkPageParseBlock(XLogReaderState *record, if (blockstate == NULL) { return NULL; } - XLogRecSetBlockDataState(record, BTREE_UNLINK_PAGE_CUR_PAGE_NUM, blockstate); + XLogRecSetBlockDataState(record, BTREE_UNLINK_PAGE_CUR_PAGE_NUM, blockstate, BLOCK_DATA_MAIN_DATA_TYPE, true); if (XLogRecHasBlockRef(record, BTREE_UNLINK_PAGE_CHILD_NUM)) { (*blocknum)++; @@ -762,7 +769,7 @@ static XLogRecParseState *BtreeXlogUnlinkPageParseBlock(XLogReaderState *record, if (blockstate == NULL) { return NULL; } - XLogRecSetBlockDataState(record, BTREE_UNLINK_PAGE_CHILD_NUM, blockstate); + XLogRecSetBlockDataState(record, BTREE_UNLINK_PAGE_CHILD_NUM, blockstate, BLOCK_DATA_MAIN_DATA_TYPE, true); } /* Update metapage if needed */ @@ -772,7 +779,7 @@ static XLogRecParseState *BtreeXlogUnlinkPageParseBlock(XLogReaderState *record, if (blockstate == NULL) { return NULL; } - XLogRecSetBlockDataState(record, BTREE_UNLINK_PAGE_META_NUM, blockstate); + XLogRecSetBlockDataState(record, BTREE_UNLINK_PAGE_META_NUM, blockstate, BLOCK_DATA_MAIN_DATA_TYPE, true); } return recordstatehead; @@ -789,7 +796,7 @@ static XLogRecParseState *BtreeXlogNewrootParseBlock(XLogReaderState *record, ui if (recordstatehead == NULL) { return NULL; } - XLogRecSetBlockDataState(record, BTREE_NEWROOT_ORIG_BLOCK_NUM, recordstatehead); + XLogRecSetBlockDataState(record, BTREE_NEWROOT_ORIG_BLOCK_NUM, recordstatehead, BLOCK_DATA_MAIN_DATA_TYPE, true); if (xlrec->level > 0) { (*blocknum)++; @@ -797,7 +804,7 @@ static XLogRecParseState *BtreeXlogNewrootParseBlock(XLogReaderState *record, ui if (blockstate == NULL) { return NULL; } - XLogRecSetBlockDataState(record, BTREE_NEWROOT_LEFT_BLOCK_NUM, blockstate); + XLogRecSetBlockDataState(record, BTREE_NEWROOT_LEFT_BLOCK_NUM, blockstate, BLOCK_DATA_MAIN_DATA_TYPE, true); } (*blocknum)++; @@ -805,7 +812,7 @@ static XLogRecParseState *BtreeXlogNewrootParseBlock(XLogReaderState *record, ui if (blockstate == NULL) { return NULL; } - XLogRecSetBlockDataState(record, BTREE_NEWROOT_META_BLOCK_NUM, blockstate); + XLogRecSetBlockDataState(record, BTREE_NEWROOT_META_BLOCK_NUM, blockstate, BLOCK_DATA_MAIN_DATA_TYPE, true); return recordstatehead; } @@ -813,24 +820,16 @@ static XLogRecParseState *BtreeXlogNewrootParseBlock(XLogReaderState *record, ui static XLogRecParseState *BtreeXlogReusePageParseBlock(XLogReaderState *record, uint32 *blocknum) { XLogRecParseState *recordstatehead = NULL; - xl_btree_reuse_page *xlrec = (xl_btree_reuse_page *)XLogRecGetData(record); - *blocknum = 0; - if (g_supportHotStandby) { - (*blocknum)++; - XLogParseBufferAllocListFunc(record, &recordstatehead, NULL); - if (recordstatehead == NULL) { - return NULL; - } - - RelFileNode rnode; - RelFileNodeCopy(rnode, xlrec->node, XLogRecGetBucketId(record)); + *blocknum = 1; - RelFileNodeForkNum filenode = - RelFileNodeForkNumFill(&rnode, InvalidBackendId, InvalidForkNumber, InvalidBlockNumber); - XLogRecSetBlockCommonState(record, BLOCK_DATA_INVALIDMSG_TYPE, filenode, recordstatehead); - XLogRecSetInvalidMsgState(&recordstatehead->blockparse.extra_rec.blockinvalidmsg, xlrec->latestRemovedXid); + XLogParseBufferAllocListFunc(record, &recordstatehead, NULL); + if (recordstatehead == NULL) { + return NULL; } + + XLogRecSetBlockDataState(record, BTREE_REUSE_PAGE_BLOCK_NUM, recordstatehead, BLOCK_DATA_MAIN_DATA_TYPE, true); + return recordstatehead; } @@ -1285,6 +1284,11 @@ void BtreeRedoDataBlock(XLogBlockHead *blockhead, XLogBlockDataParse *blockdatar case XLOG_BTREE_NEWROOT: BtreeXlogNewrootBlock(blockhead, blockdatarec, bufferinfo); break; + case XLOG_BTREE_REUSE_PAGE: + if (!(IS_EXRTO_STANDBY_READ && g_instance.attr.attr_storage.enable_exrto_standby_read_opt)) { + ereport(PANIC, (errmsg("btree_redo_block: unknown op code %u", info))); + } + break; default: ereport(PANIC, (errmsg("btree_redo_block: unknown op code %u", info))); } diff --git a/src/gausskernel/storage/access/redo/redo_ubtxlog.cpp b/src/gausskernel/storage/access/redo/redo_ubtxlog.cpp index f5fc6446a..0d8090940 100644 --- a/src/gausskernel/storage/access/redo/redo_ubtxlog.cpp +++ b/src/gausskernel/storage/access/redo/redo_ubtxlog.cpp @@ -591,9 +591,12 @@ XLogRecParseState *UBTreeXlogInsertParseBlock(XLogReaderState *record, uint32 *b if (recordstatehead == NULL) { return NULL; } - XLogRecSetBlockDataState(record, BTREE_INSERT_ORIG_BLOCK_NUM, recordstatehead); - if (info != XLOG_UBTREE_INSERT_LEAF) { + if (info == XLOG_UBTREE_INSERT_LEAF) { + XLogRecSetBlockDataState(record, BTREE_INSERT_ORIG_BLOCK_NUM, recordstatehead); + } else { + XLogRecSetBlockDataState(record, BTREE_INSERT_ORIG_BLOCK_NUM, recordstatehead, BLOCK_DATA_MAIN_DATA_TYPE, + true); (*blocknum)++; XLogParseBufferAllocListFunc(record, &blockstate, recordstatehead); if (blockstate == NULL) { @@ -608,7 +611,7 @@ XLogRecParseState *UBTreeXlogInsertParseBlock(XLogReaderState *record, uint32 *b if (blockstate == NULL) { return NULL; } - XLogRecSetBlockDataState(record, BTREE_INSERT_META_BLOCK_NUM, blockstate); + XLogRecSetBlockDataState(record, BTREE_INSERT_META_BLOCK_NUM, blockstate, BLOCK_DATA_MAIN_DATA_TYPE, true); } return recordstatehead; @@ -634,7 +637,7 @@ static XLogRecParseState *UBTreeXlogSplitParseBlock(XLogReaderState *record, uin if (recordstatehead == NULL) { return NULL; } - XLogRecSetBlockDataState(record, BTREE_SPLIT_LEFT_BLOCK_NUM, recordstatehead); + XLogRecSetBlockDataState(record, BTREE_SPLIT_LEFT_BLOCK_NUM, recordstatehead, BLOCK_DATA_MAIN_DATA_TYPE, true); XLogRecSetAuxiBlkNumState(&recordstatehead->blockparse.extra_rec.blockdatarec, rightsib, InvalidForkNumber); (*blocknum)++; @@ -651,7 +654,7 @@ static XLogRecParseState *UBTreeXlogSplitParseBlock(XLogReaderState *record, uin if (blockstate == NULL) { return NULL; } - XLogRecSetBlockDataState(record, BTREE_SPLIT_RIGHTNEXT_BLOCK_NUM, blockstate); + XLogRecSetBlockDataState(record, BTREE_SPLIT_RIGHTNEXT_BLOCK_NUM, blockstate, BLOCK_DATA_MAIN_DATA_TYPE, true); XLogRecSetAuxiBlkNumState(&blockstate->blockparse.extra_rec.blockdatarec, rightsib, InvalidForkNumber); } @@ -670,7 +673,6 @@ static XLogRecParseState *UBTreeXlogSplitParseBlock(XLogReaderState *record, uin static XLogRecParseState *UBTreeXlogVacuumParseBlock(XLogReaderState *record, uint32 *blocknum) { XLogRecParseState *recordstatehead = NULL; - XLogRecParseState *blockstate = NULL; *blocknum = 1; XLogParseBufferAllocListFunc(record, &recordstatehead, NULL); @@ -678,28 +680,7 @@ static XLogRecParseState *UBTreeXlogVacuumParseBlock(XLogReaderState *record, ui return NULL; } - XLogRecSetBlockDataState(record, BTREE_VACUUM_ORIG_BLOCK_NUM, recordstatehead); - - if (g_supportHotStandby) { - BlockNumber thisblkno = InvalidBlockNumber; - RelFileNode thisrnode = ((RelFileNode) {0, 0, 0, -1}); - - xl_btree_vacuum *xlrec = (xl_btree_vacuum *)XLogRecGetData(record); - XLogRecGetBlockTag(record, BTREE_VACUUM_ORIG_BLOCK_NUM, &thisrnode, NULL, &thisblkno); - - if ((xlrec->lastBlockVacuumed + 1) < thisblkno) { - (*blocknum)++; - XLogParseBufferAllocListFunc(record, &blockstate, recordstatehead); - if (blockstate == NULL) { - return NULL; - } - - RelFileNodeForkNum filenode = RelFileNodeForkNumFill(&thisrnode, InvalidBackendId, MAIN_FORKNUM, thisblkno); - XLogRecSetBlockCommonState(record, BLOCK_DATA_VACUUM_PIN_TYPE, filenode, blockstate); - XLogRecSetPinVacuumState(&blockstate->blockparse.extra_rec.blockvacuumpin, xlrec->lastBlockVacuumed); - } - } - + XLogRecSetBlockDataState(record, BTREE_VACUUM_ORIG_BLOCK_NUM, recordstatehead, BLOCK_DATA_MAIN_DATA_TYPE, true); return recordstatehead; } @@ -713,7 +694,7 @@ static XLogRecParseState *UBTreeXlogDeleteParseBlock(XLogReaderState *record, ui return NULL; } - XLogRecSetBlockDataState(record, BTREE_DELETE_ORIG_BLOCK_NUM, recordstatehead); + XLogRecSetBlockDataState(record, BTREE_DELETE_ORIG_BLOCK_NUM, recordstatehead, BLOCK_DATA_MAIN_DATA_TYPE, true); /* for hot standby, need to reslove the conflict */ { @@ -733,14 +714,14 @@ static XLogRecParseState *UBTreeXlogMarkHalfdeadParseBlock(XLogReaderState *reco return NULL; } - XLogRecSetBlockDataState(record, BTREE_HALF_DEAD_PARENT_PAGE_NUM, recordstatehead); + XLogRecSetBlockDataState(record, BTREE_HALF_DEAD_PARENT_PAGE_NUM, recordstatehead, BLOCK_DATA_MAIN_DATA_TYPE, true); (*blocknum)++; XLogParseBufferAllocListFunc(record, &blockstate, recordstatehead); if (blockstate == NULL) { return NULL; } - XLogRecSetBlockDataState(record, BTREE_HALF_DEAD_LEAF_PAGE_NUM, blockstate); + XLogRecSetBlockDataState(record, BTREE_HALF_DEAD_LEAF_PAGE_NUM, blockstate, BLOCK_DATA_MAIN_DATA_TYPE, true); return recordstatehead; } @@ -758,7 +739,7 @@ static XLogRecParseState *UBTreeXlogUnlinkPageParseBlock(XLogReaderState *record return NULL; } - XLogRecSetBlockDataState(record, BTREE_UNLINK_PAGE_RIGHT_NUM, recordstatehead); + XLogRecSetBlockDataState(record, BTREE_UNLINK_PAGE_RIGHT_NUM, recordstatehead, BLOCK_DATA_MAIN_DATA_TYPE, true); if (xlrec->leftsib != P_NONE) { (*blocknum)++; @@ -766,7 +747,7 @@ static XLogRecParseState *UBTreeXlogUnlinkPageParseBlock(XLogReaderState *record if (blockstate == NULL) { return NULL; } - XLogRecSetBlockDataState(record, BTREE_UNLINK_PAGE_LEFT_NUM, blockstate); + XLogRecSetBlockDataState(record, BTREE_UNLINK_PAGE_LEFT_NUM, blockstate, BLOCK_DATA_MAIN_DATA_TYPE, true); } (*blocknum)++; @@ -774,7 +755,7 @@ static XLogRecParseState *UBTreeXlogUnlinkPageParseBlock(XLogReaderState *record if (blockstate == NULL) { return NULL; } - XLogRecSetBlockDataState(record, BTREE_UNLINK_PAGE_CUR_PAGE_NUM, blockstate); + XLogRecSetBlockDataState(record, BTREE_UNLINK_PAGE_CUR_PAGE_NUM, blockstate, BLOCK_DATA_MAIN_DATA_TYPE, true); if (XLogRecHasBlockRef(record, BTREE_UNLINK_PAGE_CHILD_NUM)) { (*blocknum)++; @@ -782,7 +763,7 @@ static XLogRecParseState *UBTreeXlogUnlinkPageParseBlock(XLogReaderState *record if (blockstate == NULL) { return NULL; } - XLogRecSetBlockDataState(record, BTREE_UNLINK_PAGE_CHILD_NUM, blockstate); + XLogRecSetBlockDataState(record, BTREE_UNLINK_PAGE_CHILD_NUM, blockstate, BLOCK_DATA_MAIN_DATA_TYPE, true); } /* Update metapage if needed */ @@ -792,7 +773,7 @@ static XLogRecParseState *UBTreeXlogUnlinkPageParseBlock(XLogReaderState *record if (blockstate == NULL) { return NULL; } - XLogRecSetBlockDataState(record, BTREE_UNLINK_PAGE_META_NUM, blockstate); + XLogRecSetBlockDataState(record, BTREE_UNLINK_PAGE_META_NUM, blockstate, BLOCK_DATA_MAIN_DATA_TYPE, true); } return recordstatehead; @@ -809,7 +790,7 @@ static XLogRecParseState *UBTreeXlogNewrootParseBlock(XLogReaderState *record, u if (recordstatehead == NULL) { return NULL; } - XLogRecSetBlockDataState(record, BTREE_NEWROOT_ORIG_BLOCK_NUM, recordstatehead); + XLogRecSetBlockDataState(record, BTREE_NEWROOT_ORIG_BLOCK_NUM, recordstatehead, BLOCK_DATA_MAIN_DATA_TYPE, true); if (xlrec->level > 0) { (*blocknum)++; @@ -817,7 +798,7 @@ static XLogRecParseState *UBTreeXlogNewrootParseBlock(XLogReaderState *record, u if (blockstate == NULL) { return NULL; } - XLogRecSetBlockDataState(record, BTREE_NEWROOT_LEFT_BLOCK_NUM, blockstate); + XLogRecSetBlockDataState(record, BTREE_NEWROOT_LEFT_BLOCK_NUM, blockstate, BLOCK_DATA_MAIN_DATA_TYPE, true); } (*blocknum)++; @@ -825,7 +806,7 @@ static XLogRecParseState *UBTreeXlogNewrootParseBlock(XLogReaderState *record, u if (blockstate == NULL) { return NULL; } - XLogRecSetBlockDataState(record, BTREE_NEWROOT_META_BLOCK_NUM, blockstate); + XLogRecSetBlockDataState(record, BTREE_NEWROOT_META_BLOCK_NUM, blockstate, BLOCK_DATA_MAIN_DATA_TYPE, true); return recordstatehead; } @@ -833,24 +814,14 @@ static XLogRecParseState *UBTreeXlogNewrootParseBlock(XLogReaderState *record, u static XLogRecParseState *UBTreeXlogReusePageParseBlock(XLogReaderState *record, uint32 *blocknum) { XLogRecParseState *recordstatehead = NULL; - xl_btree_reuse_page *xlrec = (xl_btree_reuse_page *)XLogRecGetData(record); - *blocknum = 0; - if (g_supportHotStandby) { - (*blocknum)++; - XLogParseBufferAllocListFunc(record, &recordstatehead, NULL); - if (recordstatehead == NULL) { - return NULL; - } - - RelFileNode rnode; - RelFileNodeCopy(rnode, xlrec->node, XLogRecGetBucketId(record)); - - RelFileNodeForkNum filenode = - RelFileNodeForkNumFill(&rnode, InvalidBackendId, InvalidForkNumber, InvalidBlockNumber); - XLogRecSetBlockCommonState(record, BLOCK_DATA_INVALIDMSG_TYPE, filenode, recordstatehead); - XLogRecSetInvalidMsgState(&recordstatehead->blockparse.extra_rec.blockinvalidmsg, xlrec->latestRemovedXid); + *blocknum = 1; + XLogParseBufferAllocListFunc(record, &recordstatehead, NULL); + if (recordstatehead == NULL) { + return NULL; } + + XLogRecSetBlockDataState(record, BTREE_REUSE_PAGE_BLOCK_NUM, recordstatehead, BLOCK_DATA_MAIN_DATA_TYPE, true); return recordstatehead; } @@ -864,7 +835,7 @@ static XLogRecParseState* UBTreeXlogMarkDeleteParseBlock(XLogReaderState* record return NULL; } - XLogRecSetBlockDataState(record, UBTREE_MARK_DELETE_BLOCK_NUM, recordstatehead); + XLogRecSetBlockDataState(record, UBTREE_MARK_DELETE_BLOCK_NUM, recordstatehead, BLOCK_DATA_MAIN_DATA_TYPE, true); return recordstatehead; } @@ -878,7 +849,7 @@ static XLogRecParseState* UBTreeXlogPrunePageParseBlock(XLogReaderState* record, return NULL; } - XLogRecSetBlockDataState(record, UBTREE_PAGE_PRUNE_BLOCK_NUM, recordstatehead); + XLogRecSetBlockDataState(record, UBTREE_PAGE_PRUNE_BLOCK_NUM, recordstatehead, BLOCK_DATA_MAIN_DATA_TYPE, true); return recordstatehead; } @@ -892,7 +863,7 @@ static XLogRecParseState* UBTree2XlogShiftBaseParseBlock(XLogReaderState* record return NULL; } - XLogRecSetBlockDataState(record, UBTREE2_BASE_SHIFT_BLOCK_NUM, recordstatehead); + XLogRecSetBlockDataState(record, UBTREE2_BASE_SHIFT_BLOCK_NUM, recordstatehead, BLOCK_DATA_MAIN_DATA_TYPE, true); return recordstatehead; } @@ -973,7 +944,7 @@ XLogRecParseState *UBTree2XlogFreezeParseBlock(XLogReaderState *record, uint32 * if (recordstatehead == NULL) { return NULL; } - XLogRecSetBlockDataState(record, UBTREE2_FREEZE_BLOCK_NUM, recordstatehead); + XLogRecSetBlockDataState(record, UBTREE2_FREEZE_BLOCK_NUM, recordstatehead, BLOCK_DATA_MAIN_DATA_TYPE, true); return recordstatehead; } @@ -1603,7 +1574,13 @@ void UBTreeRedoDataBlock(XLogBlockHead* blockhead, XLogBlockDataParse* blockdata case XLOG_UBTREE_PRUNE_PAGE: UBTreeXlogPrunePageBlock(blockhead, blockdatarec, bufferinfo); break; + case XLOG_UBTREE_REUSE_PAGE: + if (!(IS_EXRTO_STANDBY_READ && g_instance.attr.attr_storage.enable_exrto_standby_read_opt)) { + ereport(PANIC, (errmsg("UBTreeRedoDataBlock: unknown op code %u", info))); + } + break; default: ereport(PANIC, (errmsg("UBTreeRedoDataBlock: unknown op code %u", info))); + break; } } diff --git a/src/gausskernel/storage/access/redo/redo_xlogutils.cpp b/src/gausskernel/storage/access/redo/redo_xlogutils.cpp index 53f86af4d..30ee49700 100644 --- a/src/gausskernel/storage/access/redo/redo_xlogutils.cpp +++ b/src/gausskernel/storage/access/redo/redo_xlogutils.cpp @@ -413,7 +413,7 @@ void XLogRecSetBlockDataStateContent(XLogReaderState *record, uint32 blockid, XL } void XLogRecSetBlockDataState(XLogReaderState *record, uint32 blockid, XLogRecParseState *recordblockstate, - XLogBlockParseEnum type) + XLogBlockParseEnum type, bool is_conflict_type) { Assert(XLogRecHasBlockRef(record, blockid)); DecodedBkpBlock *decodebkp = &(record->blocks[blockid]); @@ -431,6 +431,7 @@ void XLogRecSetBlockDataState(XLogReaderState *record, uint32 blockid, XLogRecPa XLogBlockDataParse *blockdatarec = &(recordblockstate->blockparse.extra_rec.blockdatarec); XLogRecSetBlockDataStateContent(record, blockid, blockdatarec); + recordblockstate->blockparse.blockhead.is_conflict_type = is_conflict_type; } void XLogRecSetAuxiBlkNumState(XLogBlockDataParse *blockdatarec, BlockNumber auxilaryblkn1, BlockNumber auxilaryblkn2) @@ -466,7 +467,7 @@ void XLogRecSetVmBlockState(XLogReaderState *record, uint32 blockid, XLogRecPars XLogBlockVmParse *blockvm = &(recordblockstate->blockparse.extra_rec.blockvmrec); blockvm->heapBlk = heapBlk; - + recordblockstate->blockparse.blockhead.is_conflict_type = true; } void GetXlUndoHeaderExtraData(char **currLogPtr, XlUndoHeaderExtra *xlundohdrextra, uint8 flag) @@ -807,6 +808,11 @@ void XLogUpdateCopyedBlockState(XLogRecParseState *recordblockstate, XLogBlockPa recordblockstate->blockparse.blockhead.bucketNode = (int2)bucketNode; } +void wal_rec_set_clean_up_info_state(WalCleanupInfoParse *parse_state, TransactionId removed_xid) +{ + parse_state->removed_xid = removed_xid; +} + void XLogRecSetBlockDdlState(XLogBlockDdlParse *blockddlstate, uint32 blockddltype, char *mainData, int rels, bool compress, uint32 mainDataLen) { @@ -1500,11 +1506,13 @@ void XLogBlockDdlDoSmgrAction(XLogBlockHead *blockhead, void *blockrecbody, Redo smgr_redo_create(rnode, blockhead->forknum, blockddlrec->mainData); break; case BLOCK_DDL_TRUNCATE_RELNODE: { - TransactionId latest_removed_xid = InvalidTransactionId; - if (blockddlrec->mainDataLen == TRUNCATE_CONTAIN_XID_SIZE) { - latest_removed_xid = ((xl_smgr_truncate_compress*)blockddlrec->mainData)->latest_removed_xid; - } - xlog_block_smgr_redo_truncate(rnode, blockhead->blkno, blockhead->end_ptr, latest_removed_xid); + RelFileNode rel_node; + rel_node.spcNode = blockhead->spcNode; + rel_node.dbNode = blockhead->dbNode; + rel_node.relNode = blockhead->relNode; + rel_node.bucketNode = blockhead->bucketNode; + rel_node.opt = blockhead->opt; + XLogTruncateRelation(rel_node, blockhead->forknum, blockhead->blkno); break; } case BLOCK_DDL_DROP_RELNODE: { @@ -1729,6 +1737,22 @@ void XLogSynAllBuffer() } } +bool need_restore_new_page_version(XLogRecParseState *redo_block_state) +{ + if (!IsHeap2Clean(&redo_block_state->blockparse.blockhead)) { + return true; + } + + TransactionId recyle_xmin = pg_atomic_read_u64(&g_instance.comm_cxt.predo_cxt.exrto_recyle_xmin); + xl_heap_clean *xl_clean_rec = + (xl_heap_clean *)XLogBlockDataGetMainData(&redo_block_state->blockparse.extra_rec.blockdatarec, NULL); + if (TransactionIdPrecedes(xl_clean_rec->latestRemovedXid, recyle_xmin)) { + return false; + } + + return true; +} + bool XLogBlockRedoForExtremeRTO(XLogRecParseState *redoblocktate, RedoBufferInfo *bufferinfo, bool notfound, RedoTimeCost &readBufCost, RedoTimeCost &redoCost) { @@ -1768,15 +1792,29 @@ bool XLogBlockRedoForExtremeRTO(XLogRecParseState *redoblocktate, RedoBufferInfo } if ((block_valid != BLOCK_DATA_UNDO_TYPE) && g_instance.attr.attr_storage.EnableHotStandby && - IsDefaultExtremeRtoMode() && XLByteLT(PageGetLSN(bufferinfo->pageinfo.page), blockhead->end_ptr)) { - if (bufferinfo->blockinfo.forknum >= EXRTO_FORK_NUM) { + IsDefaultExtremeRtoMode() && XLByteLT(PageGetLSN(bufferinfo->pageinfo.page), blockhead->end_ptr) && + !IsSegmentFileNode(bufferinfo->blockinfo.rnode)) { + if (unlikely(bufferinfo->blockinfo.forknum >= EXRTO_FORK_NUM)) { ereport(PANIC, (errmsg("forknum is illegal: %d", bufferinfo->blockinfo.forknum))); } BufferTag buf_tag; - INIT_BUFFERTAG(buf_tag, bufferinfo->blockinfo.rnode, - bufferinfo->blockinfo.forknum, bufferinfo->blockinfo.blkno); - extreme_rto_standby_read::insert_lsn_to_block_info(&extreme_rto::g_redoWorker->standby_read_meta_info, buf_tag, - bufferinfo->pageinfo.page, blockhead->start_ptr); + INIT_BUFFERTAG( + buf_tag, bufferinfo->blockinfo.rnode, bufferinfo->blockinfo.forknum, bufferinfo->blockinfo.blkno); + + if (g_instance.attr.attr_storage.enable_exrto_standby_read_opt) { + if (blockhead->is_conflict_type && need_restore_new_page_version(redoblocktate)) { + extreme_rto_standby_read::insert_lsn_to_block_info_for_opt( + &extreme_rto::g_redoWorker->standby_read_meta_info, + buf_tag, + bufferinfo->pageinfo.page, + blockhead->start_ptr); + } + } else { + extreme_rto_standby_read::insert_lsn_to_block_info(&extreme_rto::g_redoWorker->standby_read_meta_info, + buf_tag, + bufferinfo->pageinfo.page, + blockhead->start_ptr); + } } if (redoaction != BLK_DONE) { diff --git a/src/gausskernel/storage/access/redo/standby_read/Makefile b/src/gausskernel/storage/access/redo/standby_read/Makefile index 9d1fc6488..e1b7b2660 100644 --- a/src/gausskernel/storage/access/redo/standby_read/Makefile +++ b/src/gausskernel/storage/access/redo/standby_read/Makefile @@ -32,6 +32,6 @@ ifneq "$(MAKECMDGOALS)" "clean" endif endif endif -OBJS = base_page_proc.o block_info_proc.o lsn_info_double_list.o lsn_info_proc.o standby_read_interface.o standby_read_delay_ddl.o +OBJS = base_page_proc.o block_info_proc.o lsn_info_double_list.o lsn_info_proc.o standby_read_interface.o standby_read_delay_ddl.o standby_read_proc.o include $(top_srcdir)/src/gausskernel/common.mk diff --git a/src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp b/src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp index 08fefe6ce..c5bc9a5c9 100644 --- a/src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp @@ -127,7 +127,7 @@ void init_block_info(BlockMetaInfo* block_info, XLogRecPtr max_lsn) } void insert_lsn_to_block_info( - StandbyReadMetaInfo* meta_info, const BufferTag& buf_tag, const Page base_page, XLogRecPtr next_lsn) + StandbyReadMetaInfo *meta_info, const BufferTag &buf_tag, const Page base_page, XLogRecPtr next_lsn) { Buffer block_info_buf = InvalidBuffer; BlockMetaInfo* block_info = get_block_meta_info_by_relfilenode(buf_tag, NULL, RBM_ZERO_ON_ERROR, &block_info_buf); @@ -151,24 +151,76 @@ void insert_lsn_to_block_info( if (block_info->record_num == 0 || (block_info->record_num % (uint32)g_instance.attr.attr_storage.base_page_saved_interval) == 0) { - insert_base_page_to_lsn_info(meta_info, &block_info->lsn_info_list, &block_info->base_page_info_list, buf_tag, - base_page, current_page_lsn, next_lsn); + insert_base_page_to_lsn_info(meta_info, + &block_info->lsn_info_list, + &block_info->base_page_info_list, + buf_tag, + base_page, + current_page_lsn, + next_lsn); } else { insert_lsn_to_lsn_info(meta_info, &block_info->lsn_info_list, next_lsn); } + ++(block_info->record_num); Assert(block_info->max_lsn <= next_lsn); block_info->max_lsn = next_lsn; + standby_read_meta_page_set_lsn(page, next_lsn); + MarkBufferDirty(block_info_buf); + UnlockReleaseBuffer(block_info_buf); +} - ++(block_info->record_num); +void insert_lsn_to_block_info_for_opt( + StandbyReadMetaInfo *meta_info, const BufferTag &buf_tag, const Page base_page, XLogRecPtr next_lsn) +{ + Buffer block_info_buf = InvalidBuffer; + BlockMetaInfo *block_info = get_block_meta_info_by_relfilenode(buf_tag, NULL, RBM_ZERO_ON_ERROR, &block_info_buf); + if (unlikely(block_info == NULL || block_info_buf == InvalidBuffer)) { + ereport(PANIC, + (errmsg("insert lsn failed,block invalid %u/%u/%u %d %u", + buf_tag.rnode.spcNode, + buf_tag.rnode.dbNode, + buf_tag.rnode.relNode, + buf_tag.forkNum, + buf_tag.blockNum))); + } +#ifdef ENABLE_UT + Page page = get_page_from_buffer(block_info_buf); +#else + Page page = BufferGetPage(block_info_buf); +#endif + XLogRecPtr current_page_lsn = PageGetLSN(base_page); + /* if block is invalid or block is valid but all the lsn object of this block has been recycled(no data in lsn info + * files belongs to this block), we reset this block + */ + if (!is_block_meta_info_valid(block_info) || + block_info->lsn_info_list.prev < meta_info->lsn_table_recyle_position) { + if (!is_block_info_page_valid((BlockInfoPageHeader *)page)) { + block_info_page_init(page); + } + + init_block_info(block_info, current_page_lsn); + } + + insert_base_page_to_lsn_info(meta_info, + &block_info->lsn_info_list, + &block_info->base_page_info_list, + buf_tag, + base_page, + current_page_lsn, + next_lsn); + ++(block_info->record_num); + Assert(block_info->max_lsn <= next_lsn); + block_info->max_lsn = next_lsn; standby_read_meta_page_set_lsn(page, next_lsn); MarkBufferDirty(block_info_buf); UnlockReleaseBuffer(block_info_buf); } -StandbyReadRecyleState recyle_block_info( - const BufferTag& buf_tag, LsnInfoPosition base_page_info_pos, XLogRecPtr next_base_page_lsn, XLogRecPtr recyle_lsn) +StandbyReadRecyleState recyle_block_info(const BufferTag &buf_tag, LsnInfoPosition base_page_info_pos, + XLogRecPtr next_base_page_lsn, XLogRecPtr recyle_lsn, + XLogRecPtr *block_info_max_lsn) { Buffer buffer = InvalidBuffer; BlockMetaInfo* block_meta_info = get_block_meta_info_by_relfilenode(buf_tag, NULL, RBM_NORMAL, &buffer); @@ -181,6 +233,7 @@ StandbyReadRecyleState recyle_block_info( } StandbyReadRecyleState stat = STANDBY_READ_RECLYE_NONE; Assert(((block_meta_info->flags & BLOCK_INFO_NODE_VALID_FLAG) == BLOCK_INFO_NODE_VALID_FLAG)); + *block_info_max_lsn = block_meta_info->max_lsn; if (XLByteLT(block_meta_info->max_lsn, recyle_lsn)) { ereport(DEBUG1, (errmsg(EXRTOFORMAT("block meta recycle all %u/%u/%u %d %u, max lsn %08X/%08X, recycle lsn %08X/%08X"), @@ -263,9 +316,26 @@ bool get_page_lsn_info(const BufferTag& buf_tag, BufferAccessStrategy strategy, */ void remove_one_block_info_file(const RelFileNode rnode) { - DropRelFileNodeShareBuffers(rnode, MAIN_FORKNUM, 0); - DropRelFileNodeShareBuffers(rnode, FSM_FORKNUM, 0); - DropRelFileNodeShareBuffers(rnode, VISIBILITYMAP_FORKNUM, 0); + HTAB *relfilenode_hashtbl = g_instance.bgwriter_cxt.unlink_rel_hashtbl; + DelFileTag *entry = NULL; + bool found = false; + + LWLockAcquire(g_instance.bgwriter_cxt.rel_hashtbl_lock, LW_EXCLUSIVE); + entry = (DelFileTag*)hash_search(relfilenode_hashtbl, &rnode, HASH_ENTER, &found); + if (!found) { + entry->rnode.spcNode = rnode.spcNode; + entry->rnode.dbNode = rnode.dbNode; + entry->rnode.relNode = rnode.relNode; + entry->rnode.bucketNode = rnode.bucketNode; + entry->rnode.opt = rnode.opt; + entry->maxSegNo = 0; /* no need to forget fsyncs of segment */ + entry->fileUnlink = false; + } + LWLockRelease(g_instance.bgwriter_cxt.rel_hashtbl_lock); + + if (!found && g_instance.bgwriter_cxt.invalid_buf_proc_latch != NULL) { + SetLatch(g_instance.bgwriter_cxt.invalid_buf_proc_latch); + } SMgrRelation srel = smgropen(rnode, InvalidBackendId); smgrdounlink(srel, true); diff --git a/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp b/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp index 622774215..1eaf0d51e 100644 --- a/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp @@ -255,7 +255,21 @@ LsnInfoPosition create_base_page_info_node(StandbyReadMetaInfo *meta_info, base_page_info->base_page_position = base_page_pos; set_base_page_map_bit(page, offset); - + ereport(DEBUG1, + (errmsg("create_base_page_info_node, block is %u/%u/%u %d %u, batch_id: %u, redo_worker_id: %u" + "page lsn %lu, next lsn %lu, base page pos %lu, insert pos %lu", + buf_tag->rnode.spcNode, + buf_tag->rnode.dbNode, + buf_tag->rnode.relNode, + buf_tag->forkNum, + buf_tag->blockNum, + batch_id, + worker_id, + current_page_lsn, + next_lsn, + base_page_pos, + insert_pos))); + standby_read_meta_page_set_lsn(page, next_lsn); MarkBufferDirty(buffer); UnlockReleaseBuffer(buffer); @@ -497,11 +511,19 @@ void recycle_one_lsn_info_list(const BufferTag& buf_tag, LsnInfoPosition page_in /* retain a page version with page lsn less than recycle lsn */ XLogRecPtr next_base_page_lsn = base_page_info->next_base_page_lsn; - if (XLogRecPtrIsInvalid(next_base_page_lsn) || XLByteLT(recycle_lsn, next_base_page_lsn)) { - UnlockReleaseBuffer(buffer); - break; + if (g_instance.attr.attr_storage.enable_exrto_standby_read_opt) { + XLogRecPtr next_lsn = base_page_info->lsn_info_node.lsn[0]; + if (XLogRecPtrIsValid(next_lsn) && XLByteLE(recycle_lsn, next_lsn)) { + UnlockReleaseBuffer(buffer); + break; + } + } else { + if (XLogRecPtrIsInvalid(next_base_page_lsn) || XLByteLT(recycle_lsn, next_base_page_lsn)) { + UnlockReleaseBuffer(buffer); + break; + } } - + base_page_info->lsn_info_node.flags &= ~LSN_INFO_NODE_VALID_FLAG; page_info_pos = base_page_info->base_page_list.next; MarkBufferDirty(buffer); @@ -545,16 +567,26 @@ void invalid_base_page_list(StandbyReadMetaInfo *meta_info, Buffer buffer, uint3 } } -inline void update_recycle_lsn_per_worker(StandbyReadMetaInfo *meta_info, XLogRecPtr lsn) +inline void update_recycle_lsn_per_worker(StandbyReadMetaInfo *meta_info, XLogRecPtr base_page_lsn, + XLogRecPtr next_base_page_lsn, + XLogRecPtr block_info_max_lsn = InvalidXLogRecPtr) { - Assert(XLogRecPtrIsValid(lsn)); + Assert(XLogRecPtrIsValid(base_page_lsn)); if (XLogRecPtrIsInvalid(meta_info->recycle_lsn_per_worker) || - XLByteLT(meta_info->recycle_lsn_per_worker, lsn)) { - meta_info->recycle_lsn_per_worker = lsn; + XLByteLT(meta_info->recycle_lsn_per_worker, base_page_lsn)) { + meta_info->recycle_lsn_per_worker = base_page_lsn; } - ereport(LOG, (errmsg(EXRTOFORMAT( - "[exrto_recycle] update recycle lsn per worker , batch_id: %u, redo_id: %u, recycle lsn: %08X/%08X"), - meta_info->batch_id, meta_info->redo_id, (uint32)(lsn >> UINT64_HALF), (uint32)lsn))); + uint64 cur_base_page_recyle_segno = meta_info->base_page_recyle_position / EXRTO_BASE_PAGE_FILE_MAXSIZE; + uint64 cur_lsn_table_recyle_segno = meta_info->lsn_table_recyle_position / EXRTO_LSN_INFO_FILE_MAXSIZE; + ereport(LOG, + (errmsg(EXRTOFORMAT("[exrto_recycle] update recycle lsn per worker , batch_id: %u, redo_id: %u, recycle " + "base_page_lsn: %08X/%08X, next_base_page_lsn: %08X/%08X, block_info_max_lsn: " + "%08X/%08X, base page recycle segno: " + "%lu, lsn info recycle segno: %lu"), + meta_info->batch_id, meta_info->redo_id, (uint32)(base_page_lsn >> UINT64_HALF), + (uint32)base_page_lsn, (uint32)(next_base_page_lsn >> UINT64_HALF), (uint32)next_base_page_lsn, + (uint32)(block_info_max_lsn >> UINT64_HALF), (uint32)block_info_max_lsn, cur_base_page_recyle_segno, + cur_lsn_table_recyle_segno))); } bool recycle_one_lsn_info_page(StandbyReadMetaInfo *meta_info, XLogRecPtr recycle_lsn, @@ -598,25 +630,35 @@ bool recycle_one_lsn_info_page(StandbyReadMetaInfo *meta_info, XLogRecPtr recycl } XLogRecPtr next_base_page_lsn = base_page_info->next_base_page_lsn; *base_page_position = base_page_info->base_page_position; - if (XLogRecPtrIsValid(next_base_page_lsn) && XLByteLT(recycle_lsn, next_base_page_lsn)) { - update_recycle_lsn_per_worker(meta_info, base_page_lsn); - UnlockReleaseBuffer(buffer); - return false; + + if (g_instance.attr.attr_storage.enable_exrto_standby_read_opt) { + XLogRecPtr next_lsn = base_page_info->lsn_info_node.lsn[0]; + if (XLogRecPtrIsValid(next_lsn) && XLByteLE(recycle_lsn, next_lsn)) { + update_recycle_lsn_per_worker(meta_info, base_page_lsn, next_base_page_lsn); + UnlockReleaseBuffer(buffer); + return false; + } + } else { + if (XLogRecPtrIsValid(next_base_page_lsn) && XLByteLT(recycle_lsn, next_base_page_lsn)) { + update_recycle_lsn_per_worker(meta_info, base_page_lsn, next_base_page_lsn); + UnlockReleaseBuffer(buffer); + return false; + } } - + BufferTag buf_tag; INIT_BUFFERTAG(buf_tag, base_page_info->relfilenode, base_page_info->fork_num, base_page_info->block_num); LockBuffer(buffer, BUFFER_LOCK_UNLOCK); buffer_is_locked = false; - + + XLogRecPtr block_info_max_lsn = InvalidXLogRecPtr; StandbyReadRecyleState stat = - recyle_block_info(buf_tag, cur_base_page_info_pos, next_base_page_lsn, recycle_lsn); + recyle_block_info(buf_tag, cur_base_page_info_pos, next_base_page_lsn, recycle_lsn, &block_info_max_lsn); if (stat == STANDBY_READ_RECLYE_ALL) { invalid_base_page_list(meta_info, buffer, offset); } else if (stat == STANDBY_READ_RECLYE_NONE) { - Assert(XLogRecPtrIsInvalid(next_base_page_lsn)); - update_recycle_lsn_per_worker(meta_info, base_page_lsn); + update_recycle_lsn_per_worker(meta_info, base_page_lsn, next_base_page_lsn, block_info_max_lsn); ReleaseBuffer(buffer); return false; } @@ -639,15 +681,22 @@ void standby_read_recyle_per_workers(StandbyReadMetaInfo *meta_info, XLogRecPtr uint64 last_lsn_table_recyle_segno = meta_info->lsn_table_recyle_position / EXRTO_LSN_INFO_FILE_MAXSIZE; uint64 cur_base_page_recyle_segno, cur_lsn_table_recyle_segno; - while (meta_info->lsn_table_recyle_position + BLCKSZ < meta_info->lsn_table_next_position) { + uint64 recyled_page_len = 0; + const uint32 recyle_ratio = 32; // no need recyle so fast + while (meta_info->lsn_table_recyle_position + BLCKSZ * recyle_ratio < meta_info->lsn_table_next_position) { recycle_next_page = recycle_one_lsn_info_page(meta_info, recycle_lsn, &base_page_position); if (!recycle_next_page) { break; } /* update recycle position */ meta_info->lsn_table_recyle_position += BLCKSZ; + recyled_page_len += BLCKSZ; Assert(meta_info->lsn_table_recyle_position % BLCKSZ == 0); - RedoInterruptCallBack(); + if (recyled_page_len >= EXRTO_LSN_INFO_FILE_MAXSIZE) { + RedoInterruptCallBack(); + pg_usleep(100); // sleep 0.1ms + recyled_page_len = 0; + } } meta_info->base_page_recyle_position = base_page_position; @@ -668,4 +717,83 @@ void standby_read_recyle_per_workers(StandbyReadMetaInfo *meta_info, XLogRecPtr } } +LsnInfoPosition get_nearest_base_page_pos( + const BufferTag &buf_tag, const LsnInfoDoubleList &lsn_info_list, XLogRecPtr read_lsn) +{ + Buffer buffer; + + XLogRecPtr page_lsn = InvalidXLogRecPtr; + LsnInfoPosition base_page_pos = LSN_INFO_LIST_HEAD; + uint32 batch_id; + uint32 worker_id; + + /* get batch id and page redo worker id */ + extreme_rto::RedoItemTag redo_item_tag; + INIT_REDO_ITEM_TAG(redo_item_tag, buf_tag.rnode, buf_tag.forkNum, buf_tag.blockNum); + /* batch id and worker id start from 1 when reading a page */ + batch_id = extreme_rto::GetSlotId(buf_tag.rnode, 0, 0, (uint32)extreme_rto::get_batch_redo_num()) + 1; + worker_id = + extreme_rto::GetWorkerId(&redo_item_tag, (uint32)extreme_rto::get_page_redo_worker_num_per_manager()) + 1; + LsnInfoPosition latest_lsn_base_page_pos = lsn_info_list.prev; + + /* Find the base page with the smallest lsn and greater than read lsn from tail to head */ + do { + /* reach the end of the list */ + if (INFO_POSITION_IS_INVALID(latest_lsn_base_page_pos)) { + ereport(DEBUG1, (errmsg("can not find base page, block is %u/%u/%u %d %u, batch_id: %u, redo_worker_id: %u", + buf_tag.rnode.spcNode, buf_tag.rnode.dbNode, buf_tag.rnode.relNode, buf_tag.forkNum, + buf_tag.blockNum, batch_id, worker_id))); + break; + } + buffer = InvalidBuffer; + Page page = get_lsn_info_page(batch_id, worker_id, latest_lsn_base_page_pos, RBM_NORMAL, &buffer); + if (page == NULL || buffer == InvalidBuffer) { + ereport(ERROR, (errmsg(EXRTOFORMAT("get_nearest_base_page_pos failed, batch_id: %u, redo_id: %u, pos: %lu"), + batch_id, worker_id, latest_lsn_base_page_pos))); + } + LockBuffer(buffer, BUFFER_LOCK_SHARE); + + uint32 offset = lsn_info_postion_to_offset(latest_lsn_base_page_pos); + BasePageInfo base_page_info = (BasePageInfo)(page + offset); + + Assert(is_base_page_type(base_page_info->lsn_info_node.type)); + if (!is_base_page_type(base_page_info->lsn_info_node.type)) { + UnlockReleaseBuffer(buffer); + ereport( + ERROR, + (errmsg(EXRTOFORMAT("get_nearest_base_page_pos failed, not base page type, block is %u/%u/%u %d %u, " + "batch_id: %u, redo_id: %u, pos: %lu"), + buf_tag.rnode.spcNode, buf_tag.rnode.dbNode, buf_tag.rnode.relNode, buf_tag.forkNum, + buf_tag.blockNum, batch_id, worker_id, latest_lsn_base_page_pos))); + } + + UnlockReleaseBuffer(buffer); + page_lsn = base_page_info->lsn_info_node.lsn[0]; + LsnInfoPosition prev_lsn_base_page_pos = base_page_info->base_page_list.prev; + + if (XLByteLT(page_lsn, read_lsn)) { + break; + } + + /* the base page's lsn >= read_lsn */ + base_page_pos = base_page_info->base_page_position; + + // the last base page info + if (XLByteEQ(lsn_info_list.next, latest_lsn_base_page_pos)) { + break; + } + + latest_lsn_base_page_pos = prev_lsn_base_page_pos; + } while (true); + + if (page_lsn == InvalidXLogRecPtr || base_page_pos == LSN_INFO_LIST_HEAD) { + ereport(DEBUG1, (errmsg(EXRTOFORMAT("get_nearest_base_page_pos failed, block is %u/%u/%u/%hd/%hu %d %u, " + "batch_id: %u, redo_id: %u, pos: %lu, page_lsn: %lu"), + buf_tag.rnode.spcNode, buf_tag.rnode.dbNode, buf_tag.rnode.relNode, + buf_tag.rnode.bucketNode, buf_tag.rnode.opt, buf_tag.forkNum, buf_tag.blockNum, + batch_id, worker_id, latest_lsn_base_page_pos, page_lsn))); + } + + return base_page_pos; +} } // namespace extreme_rto_standby_read diff --git a/src/gausskernel/storage/access/redo/standby_read/standby_read_delay_ddl.cpp b/src/gausskernel/storage/access/redo/standby_read/standby_read_delay_ddl.cpp index c9adb75a6..3e1ade64b 100644 --- a/src/gausskernel/storage/access/redo/standby_read/standby_read_delay_ddl.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/standby_read_delay_ddl.cpp @@ -29,6 +29,9 @@ #include "access/extreme_rto/standby_read/block_info_meta.h" #include "access/multi_redo_api.h" #include "commands/dbcommands.h" +#include "access/slru.h" +#include "access/twophase.h" +#include "storage/procarray.h" #define DELAY_DDL_FILE_DIR "delay_ddl" #define DELAY_DDL_FILE_NAME "delay_ddl/delay_delete_info_file" @@ -36,8 +39,11 @@ typedef enum { DROP_DB_TYPE = 1, DROP_TABLE_TYPE, + TRUNCATE_CLOG, } DropDdlType; +#define ClogCtl(n) (&t_thrd.shemem_ptr_cxt.ClogCtl[CBufHashPartition(n)]) + const static uint32 MAX_NUM_PER_FILE = 0x10000; typedef struct { @@ -45,7 +51,10 @@ typedef struct { uint8 len; uint16 resvd1; uint32 resvd2; - ColFileNode node_info; + union { + ColFileNode file_info; + int64 pageno; + } node_info; XLogRecPtr lsn; pg_crc32 crc; } DelayDdlInfo; @@ -174,8 +183,8 @@ void update_delay_ddl_db(Oid db_id, Oid tablespace_id, XLogRecPtr lsn) .resvd2 = 0, }; - tmp_info.node_info.filenode.dbNode = db_id; - tmp_info.node_info.filenode.spcNode = tablespace_id; + tmp_info.node_info.file_info.filenode.dbNode = db_id; + tmp_info.node_info.file_info.filenode.spcNode = tablespace_id; tmp_info.lsn = lsn; INIT_CRC32C(tmp_info.crc); COMP_CRC32C(tmp_info.crc, (char*)&tmp_info, offsetof(DelayDdlInfo, crc)); @@ -195,7 +204,7 @@ void update_delay_ddl_files(ColFileNode* xnodes, int nrels, XLogRecPtr lsn) info_list[i].len = sizeof(DelayDdlInfo); info_list[i].resvd1 = 0; info_list[i].resvd2 = 0; - info_list[i].node_info = xnodes[i]; + info_list[i].node_info.file_info = xnodes[i]; info_list[i].lsn = lsn; INIT_CRC32C(info_list[i].crc); COMP_CRC32C(info_list[i].crc, (char*)&info_list[i], offsetof(DelayDdlInfo, crc)); @@ -236,7 +245,63 @@ void update_delay_ddl_files(ColFileNode* xnodes, int nrels, XLogRecPtr lsn) exit_state(&stat->insert_stat); } -void do_delay_ddl(DelayDdlInfo* info) +void update_delay_ddl_file_truncate_clog(XLogRecPtr lsn, int64 pageno) +{ + StandbyReadDelayDdlState *stat = &g_instance.comm_cxt.predo_cxt.standby_read_delay_ddl_stat; + enter_state(&stat->insert_stat); + uint64 insert_start = pg_atomic_read_u64(&stat->next_index_can_insert); + + char path[MAXPGPATH]; + errno_t errorno = snprintf_s(path, MAXPGPATH, MAXPGPATH - 1, DELAY_DDL_FILE_NAME "_%08X_%lX", + t_thrd.shemem_ptr_cxt.ControlFile->timeline, insert_start / MAX_NUM_PER_FILE); + securec_check_ss(errorno, "", ""); + + off_t off_set = (off_t)(insert_start % MAX_NUM_PER_FILE * sizeof(DelayDdlInfo)); + + DelayDdlInfo truncate_clog_info = {0}; + truncate_clog_info.type = TRUNCATE_CLOG; + truncate_clog_info.len = sizeof(DelayDdlInfo); + truncate_clog_info.node_info.pageno = pageno; + truncate_clog_info.lsn = lsn; + INIT_CRC32C(truncate_clog_info.crc); + COMP_CRC32C(truncate_clog_info.crc, (char*)&truncate_clog_info, offsetof(DelayDdlInfo, crc)); + FIN_CRC32C(truncate_clog_info.crc); + + if (write_delay_ddl_info(path, &truncate_clog_info, sizeof(DelayDdlInfo), off_set)) { + pg_atomic_write_u64(&stat->next_index_can_insert, insert_start + 1); + } + exit_state(&stat->insert_stat); +} + +void clog_truncate_cancel_conflicting_proc(TransactionId latest_removed_xid, XLogRecPtr lsn) +{ + const int max_check_times = 1000; + int check_times = 0; + bool conflict = true; + bool reach_max_check_times = false; + while (conflict && check_times < max_check_times) { + RedoInterruptCallBack(); + check_times++; + reach_max_check_times = (check_times == max_check_times); + conflict = proc_array_cancel_conflicting_proc(latest_removed_xid, lsn, reach_max_check_times); + } +} + +void do_truncate_clog(int64 pageno) +{ + ClogCtl(pageno)->shared->latest_page_number = pageno; + + TransactionId truncate_xid = (TransactionId)PAGE_TO_TRANSACTION_ID(pageno); + clog_truncate_cancel_conflicting_proc(truncate_xid, InvalidXLogRecPtr); + if (TransactionIdPrecedes(g_instance.undo_cxt.hotStandbyRecycleXid, truncate_xid)) { + pg_atomic_write_u64(&g_instance.undo_cxt.hotStandbyRecycleXid, truncate_xid); + } + + SimpleLruTruncate(ClogCtl(0), pageno, NUM_CLOG_PARTITIONS); + DeleteObsoleteTwoPhaseFile(pageno); +} + +void do_delay_ddl(DelayDdlInfo *info, bool is_old_delay_ddl = false) { pg_crc32c crc_check; INIT_CRC32C(crc_check); @@ -246,21 +311,45 @@ void do_delay_ddl(DelayDdlInfo* info) if (!EQ_CRC32C(crc_check, info->crc)) { ereport(WARNING, (errcode_for_file_access(), errmsg("delay ddl ,crc(%u:%u) check error, maybe is type:%u, info %u/%u/%u lsn:%lu", crc_check, info->crc, - (uint32)info->type, info->node_info.filenode.spcNode, info->node_info.filenode.dbNode, - info->node_info.filenode.relNode, info->lsn))); + (uint32)info->type, info->node_info.file_info.filenode.spcNode, info->node_info.file_info.filenode.dbNode, + info->node_info.file_info.filenode.relNode, info->lsn))); return; } if (info->type == DROP_TABLE_TYPE) { - unlink_relfiles(&info->node_info, 1); - xact_redo_log_drop_segs(&info->node_info, 1, info->lsn); + ereport(DEBUG2, + (errmodule(MOD_STANDBY_READ), + errmsg("delay ddl for table, type:%u, info %u/%u/%u lsn:%lu", + (uint32)info->type, + info->node_info.file_info.filenode.spcNode, + info->node_info.file_info.filenode.dbNode, + info->node_info.file_info.filenode.relNode, + info->lsn))); + unlink_relfiles(&info->node_info.file_info, 1, is_old_delay_ddl); + xact_redo_log_drop_segs(&info->node_info.file_info, 1, info->lsn); } else if (info->type == DROP_DB_TYPE) { - do_db_drop(info->node_info.filenode.dbNode, info->node_info.filenode.spcNode); + ereport(DEBUG2, + (errmodule(MOD_STANDBY_READ), + errmsg("delay ddl for database, type:%u, info %u/%u lsn:%lu", + (uint32)info->type, + info->node_info.file_info.filenode.spcNode, + info->node_info.file_info.filenode.dbNode, + info->lsn))); + do_db_drop(info->node_info.file_info.filenode.dbNode, info->node_info.file_info.filenode.spcNode); + } else if (info->type == TRUNCATE_CLOG) { + ereport(LOG, + (errmodule(MOD_STANDBY_READ), errmsg("delay ddl for truncate clog, pageno: %ld", info->node_info.pageno))); + UpdateMinRecoveryPoint(info->lsn, false); + do_truncate_clog(info->node_info.pageno); } else { - ereport(WARNING, (errcode_for_file_access(), - errmsg("delay ddl ,type error, maybe is type:%u, info %u/%u/%u lsn:%lu", (uint32)info->type, - info->node_info.filenode.spcNode, info->node_info.filenode.dbNode, info->node_info.filenode.relNode, - info->lsn))); + ereport(WARNING, + (errcode_for_file_access(), + errmsg("delay ddl ,type error, maybe is type:%u, info %u/%u/%u lsn:%lu", + (uint32)info->type, + info->node_info.file_info.filenode.spcNode, + info->node_info.file_info.filenode.dbNode, + info->node_info.file_info.filenode.relNode, + info->lsn))); } } @@ -293,6 +382,7 @@ void delete_by_lsn(XLogRecPtr lsn) int fd = BasicOpenFile(path, O_RDONLY | PG_BINARY, S_IRUSR | S_IWUSR); if (fd < 0) { + exit_state(&stat->delete_stat); return; } int count = read_delay_ddl_info(fd, info_list, cur_deleted * sizeof(DelayDdlInfo), (off_t)offset); @@ -326,6 +416,7 @@ void delete_by_lsn(XLogRecPtr lsn) deleted_total += cur_deleted; if (next_delete % MAX_NUM_PER_FILE == 0) { (void)unlink(path); + ereport(LOG, (errmsg("delete delay ddl file end [%s:%d:%s]", __FUNCTION__, __LINE__, path))); } RedoInterruptCallBack(); } @@ -342,6 +433,8 @@ void delete_by_table_space(Oid tablespace_id) uint64 next_delete = pg_atomic_read_u64(&stat->next_index_need_unlink); uint64 next_insert = pg_atomic_read_u64(&stat->next_index_can_insert); + ereport(LOG, (errmsg("delete_by_table_space start"))); + DelayDdlInfo* info_list = (DelayDdlInfo*)palloc0(sizeof(DelayDdlInfo) * MAX_NUM_PER_FILE); while (next_delete < next_insert) { uint32 copys = MAX_NUM_PER_FILE; @@ -363,6 +456,7 @@ void delete_by_table_space(Oid tablespace_id) if (fd < 0) { ereport(WARNING, (errmsg("delete_by_table_space: file %s could not open:%m", path))); + exit_state(&stat->delete_stat); return; } @@ -370,6 +464,7 @@ void delete_by_table_space(Oid tablespace_id) if (count <= 0) { ereport(WARNING, (errmsg("delete_by_table_space: file %s nothing deleted", path))); + exit_state(&stat->delete_stat); return; } close(fd); @@ -384,7 +479,7 @@ void delete_by_table_space(Oid tablespace_id) } for (uint32 i = 0; i < copys; ++i) { - if (info_list[i].node_info.filenode.spcNode == tablespace_id) { + if (info_list[i].node_info.file_info.filenode.spcNode == tablespace_id) { do_delay_ddl(&info_list[i]); } RedoInterruptCallBack(); @@ -394,6 +489,7 @@ void delete_by_table_space(Oid tablespace_id) } pfree(info_list); exit_state(&stat->delete_stat); + ereport(LOG, (errmsg("delete_by_table_space end"))); } void do_all_old_delay_ddl() @@ -435,6 +531,7 @@ void do_all_old_delay_ddl() } (void)unlink(path); + ereport(LOG, (errmsg("delete delay ddl file end [%s:%d:%s]", __FUNCTION__, __LINE__, path))); pfree(info_list); RedoInterruptCallBack(); } diff --git a/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp b/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp index 145fa84a1..8e307c14f 100644 --- a/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp @@ -48,11 +48,11 @@ const char* EXRTO_FILE_SUB_DIR[] = { const uint32 EXRTO_FILE_PATH_LEN = 1024; const uint32 XID_THIRTY_TWO = 32; -void make_standby_read_node(XLogRecPtr read_lsn, RelFileNode &read_node, bool is_start_lsn) +void make_standby_read_node(XLogRecPtr read_lsn, RelFileNode &read_node, bool is_start_lsn, Oid relnode) { read_node.spcNode = (Oid)(read_lsn >> 32); read_node.dbNode = (Oid)(read_lsn); - read_node.relNode = InvalidOid; // make sure it can be InvalidOid or not + read_node.relNode = relnode; read_node.opt = 0; if (is_start_lsn) { /* means read_lsn is the start ptr of xlog */ @@ -67,7 +67,7 @@ BufferDesc *alloc_standby_read_buf(const BufferTag &buf_tag, BufferAccessStrateg XLogRecPtr read_lsn, bool is_start_lsn) { RelFileNode read_node; - make_standby_read_node(read_lsn, read_node, is_start_lsn); + make_standby_read_node(read_lsn, read_node, is_start_lsn, buf_tag.rnode.relNode); BufferDesc *buf_desc = BufferAlloc(read_node, 0, buf_tag.forkNum, buf_tag.blockNum, strategy, &found, NULL); return buf_desc; @@ -78,8 +78,8 @@ Buffer get_newest_page_for_read(Relation reln, ForkNumber fork_num, BlockNumber { bool hit = false; - Buffer newest_buf = ReadBuffer_common( - reln->rd_smgr, reln->rd_rel->relpersistence, fork_num, block_num, mode, strategy, &hit, NULL); + Buffer newest_buf = + ReadBuffer_common(reln->rd_smgr, reln->rd_rel->relpersistence, fork_num, block_num, mode, strategy, &hit, NULL); if (BufferIsInvalid(newest_buf)) { return InvalidBuffer; } @@ -97,8 +97,51 @@ Buffer get_newest_page_for_read(Relation reln, ForkNumber fork_num, BlockNumber .forkNum = fork_num, .blockNum = block_num, }; + + ResourceOwnerEnlargeBuffers(t_thrd.utils_cxt.CurrentResourceOwner); + BufferDesc *buf_desc = alloc_standby_read_buf(buf_tag, strategy, hit, page_lsn, false); + + if (hit) { + UnlockReleaseBuffer(newest_buf); + return BufferDescriptorGetBuffer(buf_desc); + } + Page read_page = (Page)BufHdrGetBlock(buf_desc); + + errno_t rc = memcpy_s(read_page, BLCKSZ, newest_page, BLCKSZ); + securec_check(rc, "\0", "\0"); + + UnlockReleaseBuffer(newest_buf); + buf_desc->extra->lsn_on_disk = PageGetLSN(read_page); +#ifdef USE_ASSERT_CHECKING + buf_desc->lsn_dirty = InvalidXLogRecPtr; +#endif + + TerminateBufferIO(buf_desc, false, (BM_VALID | BM_IS_TMP_BUF)); + return BufferDescriptorGetBuffer(buf_desc); +} + +Buffer get_newest_page_for_read_new( + Relation reln, ForkNumber fork_num, BlockNumber block_num, ReadBufferMode mode, BufferAccessStrategy strategy) +{ + bool hit = false; + + Buffer newest_buf = + ReadBuffer_common(reln->rd_smgr, reln->rd_rel->relpersistence, fork_num, block_num, mode, strategy, &hit, NULL); + if (BufferIsInvalid(newest_buf)) { + return InvalidBuffer; + } + + LockBuffer(newest_buf, BUFFER_LOCK_SHARE); + Page newest_page = BufferGetPage(newest_buf); + + BufferTag buf_tag = { + .rnode = reln->rd_smgr->smgr_rnode.node, + .forkNum = fork_num, + .blockNum = block_num, + }; + ResourceOwnerEnlargeBuffers(t_thrd.utils_cxt.CurrentResourceOwner); - BufferDesc* buf_desc = alloc_standby_read_buf(buf_tag, strategy, hit, page_lsn, false); + BufferDesc *buf_desc = alloc_standby_read_buf(buf_tag, strategy, hit, PageGetLSN(newest_page), false); if (hit) { UnlockReleaseBuffer(newest_buf); @@ -108,10 +151,12 @@ Buffer get_newest_page_for_read(Relation reln, ForkNumber fork_num, BlockNumber errno_t rc = memcpy_s(read_page, BLCKSZ, newest_page, BLCKSZ); securec_check(rc, "\0", "\0"); + UnlockReleaseBuffer(newest_buf); + buf_desc->extra->lsn_on_disk = PageGetLSN(read_page); #ifdef USE_ASSERT_CHECKING - buf_desc->lsn_dirty = InvalidXLogRecPtr; + buf_desc->lsn_dirty = InvalidXLogRecPtr; #endif TerminateBufferIO(buf_desc, false, (BM_VALID | BM_IS_TMP_BUF)); @@ -121,6 +166,9 @@ Buffer get_newest_page_for_read(Relation reln, ForkNumber fork_num, BlockNumber Buffer standby_read_buf( Relation reln, ForkNumber fork_num, BlockNumber block_num, ReadBufferMode mode, BufferAccessStrategy strategy) { + if (g_instance.attr.attr_storage.enable_exrto_standby_read_opt) { + return extreme_rto_standby_read::standby_read_buf_new(reln, fork_num, block_num, mode, strategy); + } /* Open it at the smgr level */ RelationOpenSmgr(reln); // need or not ????? pgstat_count_buffer_read(reln); @@ -664,6 +712,8 @@ void dump_base_page_info_lsn_info(const BufferTag &buf_tag, LsnInfoPosition head uint32 worker_id; BasePageInfo base_page_info = NULL; Buffer buffer; + const int max_dump_item = 10000; + int cnt = 0; extreme_rto::RedoItemTag redo_item_tag; INIT_REDO_ITEM_TAG(redo_item_tag, buf_tag.rnode, buf_tag.forkNum, buf_tag.blockNum); @@ -672,6 +722,9 @@ void dump_base_page_info_lsn_info(const BufferTag &buf_tag, LsnInfoPosition head /* find fisrt base page whose lsn less than read lsn form tail to head */ do { + if (cnt > max_dump_item) { + break; + } /* reach the end of the list */ if (INFO_POSITION_IS_INVALID(head_lsn_base_page_pos)) { ereport(LOG, (errmsg("can not find base page, block is %u/%u/%u %d %u, batch_id: %u, redo_worker_id: %u", @@ -750,9 +803,18 @@ void dump_error_all_info(const RelFileNode &rnode, ForkNumber forknum, BlockNumb char *str_output = (char *)palloc0(MAXOUTPUTLEN * sizeof(char)); char *dump_filename = (char *)palloc0(MAXFILENAME * sizeof(char)); errno_t rc = snprintf_s(dump_filename + (int)strlen(dump_filename), MAXFILENAME, MAXFILENAME - 1, - "%s/%u_%u_%u_%d_%d.lsnblockinfo_dump", t_thrd.proc_cxt.DataDir, rnode.spcNode, rnode.dbNode, rnode.relNode, - forknum, blocknum); + "%s/%u_%u_%u_%d_%d.lsnblockinfo_dump", u_sess->attr.attr_common.Log_directory, + rnode.spcNode, rnode.dbNode, rnode.relNode, forknum, blocknum); securec_check_ss(rc, "\0", "\0"); + struct stat file_stat; + if (stat(dump_filename, &file_stat) == 0) { + /* file exists */ + pfree_ext(str_output); + pfree_ext(dump_filename); + buffer_in_progress_push(); + return; + } + FILE *dump_file = AllocateFile(dump_filename, PG_BINARY_W); if (dump_file == NULL) { ereport(LOG, (errmsg("can not alloc file. rnode is %u/%u/%u %d %u", buf_tag.rnode.spcNode, @@ -772,13 +834,143 @@ void dump_error_all_info(const RelFileNode &rnode, ForkNumber forknum, BlockNumb UnlockReleaseBuffer(buf); // buf was automatically locked by getting block meta info, so we need release uint result = fwrite(str_output, 1, strlen(str_output), dump_file); - if (result != strlen(str_output)) { - ereport(ERROR, (errcode(ERRCODE_FILE_WRITE_FAILED), errmsg("Cannot write into file %s!", dump_filename))); + if (result == strlen(str_output)) { + (void)fsync(fileno(dump_file)); + exrto_xlog_dump(dump_filename, dump_lsn_info_stru); + } else { + pfree_ext(str_output); + pfree_ext(dump_filename); + (void)FreeFile(dump_file); + buffer_in_progress_push(); + + ereport(ERROR, (errcode(ERRCODE_FILE_WRITE_FAILED), + errmsg("Cannot write into file %s/%u_%u_%u_%d_%u.lsnblockinfo_dump!", + u_sess->attr.attr_common.Log_directory, rnode.spcNode, rnode.dbNode, rnode.relNode, + forknum, blocknum))); } pfree_ext(str_output); - (void)FreeFile(dump_file); - exrto_xlog_dump(dump_filename, dump_lsn_info_stru); pfree_ext(dump_filename); + (void)FreeFile(dump_file); buffer_in_progress_push(); } + +Buffer standby_read_buf_new( + Relation reln, ForkNumber fork_num, BlockNumber block_num, ReadBufferMode mode, BufferAccessStrategy strategy) +{ + /* Open it at the smgr level */ + RelationOpenSmgr(reln); // need or not ????? + pgstat_count_buffer_read(reln); + pgstatCountBlocksFetched4SessionLevel(); + + if (RelationisEncryptEnable(reln)) { + reln->rd_smgr->encrypt = true; + } + + XLogRecPtr read_lsn = MAX_XLOG_REC_PTR; + if (u_sess->utils_cxt.CurrentSnapshot != NULL && XLogRecPtrIsValid(u_sess->utils_cxt.CurrentSnapshot->read_lsn)) { + read_lsn = u_sess->utils_cxt.CurrentSnapshot->read_lsn; + } else if (XLogRecPtrIsValid(t_thrd.proc->exrto_read_lsn)) { + read_lsn = t_thrd.proc->exrto_read_lsn; + } + + + Buffer read_buf = get_newest_page_for_read_new(reln, fork_num, block_num, mode, strategy); + if (unlikely(read_buf == InvalidBuffer)) { + ereport(DEBUG1, + (errmsg("couldnot get newest page buf %u/%u/%u %d %u read lsn %08X/%08X current_time: %ld " + "gen_snaptime:%ld thread_read_lsn:%08X/%08X", + reln->rd_smgr->smgr_rnode.node.spcNode, + reln->rd_smgr->smgr_rnode.node.dbNode, + reln->rd_smgr->smgr_rnode.node.relNode, + fork_num, + block_num, + (uint32)(read_lsn >> XID_THIRTY_TWO), + (uint32)read_lsn, + GetCurrentTimestamp(), + g_instance.comm_cxt.predo_cxt.exrto_snapshot->gen_snap_time, + (uint32)(t_thrd.proc->exrto_read_lsn >> XID_THIRTY_TWO), + (uint32)t_thrd.proc->exrto_read_lsn))); + return InvalidBuffer; + } + + if (XLByteLT(PageGetLSN(BufferGetPage(read_buf)), read_lsn)) { + return read_buf; + } + + BufferTag buf_tag = { + .rnode = reln->rd_smgr->smgr_rnode.node, + .forkNum = fork_num, + .blockNum = block_num, + }; + + Buffer block_info_buf; + // just lock this buffer ,so that redo worker could not modify this block info + BlockMetaInfo *block_info = + get_block_meta_info_by_relfilenode(buf_tag, NULL, RBM_ZERO_ON_ERROR, &block_info_buf, true); + if (unlikely(block_info == NULL || block_info_buf == InvalidBuffer)) { + ereport(PANIC, + (errmsg("standby_read_buf_new read block invalid %u/%u/%u/%hd/%hu %d %u", + buf_tag.rnode.spcNode, + buf_tag.rnode.dbNode, + buf_tag.rnode.relNode, + buf_tag.rnode.bucketNode, + buf_tag.rnode.opt, + buf_tag.forkNum, + buf_tag.blockNum))); + } + + if (!is_block_meta_info_valid(block_info)) { + UnlockReleaseBuffer(block_info_buf); + return read_buf; + } + + if (block_info->max_lsn < read_lsn) { + UnlockReleaseBuffer(block_info_buf); + return read_buf; + } + + // find nearest base page + LsnInfoPosition base_page_pos = get_nearest_base_page_pos(buf_tag, block_info->base_page_info_list, read_lsn); + if (base_page_pos == LSN_INFO_LIST_HEAD) { + UnlockReleaseBuffer(block_info_buf); + return read_buf; + } + UnlockReleaseBuffer(read_buf); + + extreme_rto::RedoItemTag redo_item_tag; + INIT_REDO_ITEM_TAG(redo_item_tag, buf_tag.rnode, buf_tag.forkNum, buf_tag.blockNum); + const uint32 worker_num_per_mng = (uint32)extreme_rto::get_page_redo_worker_num_per_manager(); + /* batch id and worker id start from 1 when reading a page */ + uint32 batch_id = extreme_rto::GetSlotId(buf_tag.rnode, 0, 0, (uint32)extreme_rto::get_batch_redo_num()) + 1; + uint32 redo_worker_id = extreme_rto::GetWorkerId(&redo_item_tag, worker_num_per_mng) + 1; + + Buffer base_page_buffer = buffer_read_base_page(batch_id, redo_worker_id, base_page_pos, RBM_NORMAL); + bool hit = false; + LockBuffer(base_page_buffer, BUFFER_LOCK_SHARE); + ResourceOwnerEnlargeBuffers(t_thrd.utils_cxt.CurrentResourceOwner); + + Page base_page = BufferGetPage(base_page_buffer); + XLogRecPtr base_page_lsn = PageGetLSN(base_page); + BufferDesc *buf_desc = alloc_standby_read_buf(buf_tag, strategy, hit, base_page_lsn, false); + + if (hit) { + UnlockReleaseBuffer(block_info_buf); + UnlockReleaseBuffer(base_page_buffer); + return BufferDescriptorGetBuffer(buf_desc); + } + + Page read_page = (Page)BufHdrGetBlock(buf_desc); + errno_t rc = memcpy_s(read_page, BLCKSZ, base_page, BLCKSZ); + securec_check(rc, "\0", "\0"); + + buf_desc->extra->lsn_on_disk = PageGetLSN(read_page); +#ifdef USE_ASSERT_CHECKING + buf_desc->lsn_dirty = InvalidXLogRecPtr; +#endif + + TerminateBufferIO(buf_desc, false, (BM_VALID | BM_IS_TMP_BUF)); + UnlockReleaseBuffer(block_info_buf); + UnlockReleaseBuffer(base_page_buffer); + return BufferDescriptorGetBuffer(buf_desc); } +} // namespace extreme_rto_standby_read diff --git a/src/gausskernel/storage/access/redo/standby_read/standby_read_proc.cpp b/src/gausskernel/storage/access/redo/standby_read/standby_read_proc.cpp new file mode 100644 index 000000000..c4f6231ab --- /dev/null +++ b/src/gausskernel/storage/access/redo/standby_read/standby_read_proc.cpp @@ -0,0 +1,423 @@ +/* + * Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * standby_read_proc.cpp + * + * IDENTIFICATION + * src/gausskernel/storage/access/redo/standby_read/standby_read_proc.cpp + * + * ------------------------------------------------------------------------- + */ + +#include "access/extreme_rto/page_redo.h" +#include "access/extreme_rto/standby_read/block_info_meta.h" +#include "access/extreme_rto/standby_read/lsn_info_meta.h" +#include "access/extreme_rto/standby_read/standby_read_base.h" +#include "access/multi_redo_api.h" +#include "access/extreme_rto/dispatcher.h" +#include "storage/procarray.h" +#include "replication/walreceiver.h" + +inline void invalid_msg_leak_warning(XLogRecPtr trxn_lsn) +{ + if (t_thrd.page_redo_cxt.invalid_msg.valid) { + ereport(WARNING, + (errmsg(EXRTOFORMAT("[exrto_generate_snapshot] not send invalid msg: %08X/%08X"), + (uint32)(trxn_lsn >> UINT64_HALF), + (uint32)trxn_lsn))); + } +} + +void exrto_generate_snapshot(XLogRecPtr trxn_lsn) +{ + if (!g_instance.attr.attr_storage.EnableHotStandby) { + return; + } + + ExrtoSnapshot exrto_snapshot = g_instance.comm_cxt.predo_cxt.exrto_snapshot; + /* + * do not generate the same snapshot repeatedly. + */ + if (XLByteLE(trxn_lsn, exrto_snapshot->read_lsn)) { + invalid_msg_leak_warning(trxn_lsn); + return; + } + + TransactionId xmin; + TransactionId xmax; + CommitSeqNo snapshot_csn; + + exrto_get_snapshot_data(xmin, xmax, snapshot_csn); + (void)LWLockAcquire(ExrtoSnapshotLock, LW_EXCLUSIVE); + exrto_snapshot->snapshot_csn = snapshot_csn; + exrto_snapshot->xmin = xmin; + exrto_snapshot->xmax = xmax; + exrto_snapshot->read_lsn = trxn_lsn; + send_delay_invalid_message(); + LWLockRelease(ExrtoSnapshotLock); +} + +void exrto_read_snapshot(Snapshot snapshot) +{ + if ((!is_exrto_standby_read_worker()) || u_sess->proc_cxt.clientIsCMAgent || dummyStandbyMode) { + return; + } + + ExrtoSnapshot exrto_snapshot = g_instance.comm_cxt.predo_cxt.exrto_snapshot; + bool retry_get = false; + const static uint64 WAIT_COUNT = 0x7FFFF; + uint64 retry_count = 0; + t_thrd.pgxact->xmin = InvalidTransactionId; + t_thrd.proc->exrto_min = InvalidXLogRecPtr; +RETRY_GET: + if (retry_get) { + CHECK_FOR_INTERRUPTS(); + pg_usleep(100L); + } + retry_count++; + if ((retry_count & WAIT_COUNT) == WAIT_COUNT) { + ereport(LOG, + (errmsg("retry to get exrto-standby-read snapshot, standby_redo_cleanup_xmin = %lu, " + "standby_redo_cleanup_xmin_lsn = %08X/%08X, " + "exrto_snapshot->xmin = %lu, read_lsn = %08X/%08X", + t_thrd.xact_cxt.ShmemVariableCache->standbyRedoCleanupXmin, + (uint32)(t_thrd.xact_cxt.ShmemVariableCache->standbyRedoCleanupXminLsn >> UINT64_HALF), + (uint32)t_thrd.xact_cxt.ShmemVariableCache->standbyRedoCleanupXminLsn, exrto_snapshot->xmin, + (uint32)(exrto_snapshot->read_lsn >> UINT64_HALF), (uint32)exrto_snapshot->read_lsn))); + } + (void)LWLockAcquire(ExrtoSnapshotLock, LW_SHARED); + if (XLByteEQ(exrto_snapshot->read_lsn, 0)) { + LWLockRelease(ExrtoSnapshotLock); + ereport(ERROR, (errmsg("could not get a valid snapshot with extreme rto"))); + } + + /* In exrto_standby_read_opt mode, getting a snapshot needs to wait for the cleanup-info xlog to be processed. */ + if (g_instance.attr.attr_storage.enable_exrto_standby_read_opt) { + LWLockAcquire(ProcArrayLock, LW_SHARED); + bool condition = + (exrto_snapshot->xmin <= + t_thrd.xact_cxt.ShmemVariableCache->standbyRedoCleanupXmin) && + (t_thrd.xact_cxt.ShmemVariableCache->standbyRedoCleanupXminLsn > exrto_snapshot->read_lsn); + LWLockRelease(ProcArrayLock); + if (condition) { + retry_get = true; + LWLockRelease(ExrtoSnapshotLock); + goto RETRY_GET; + } + } + + snapshot->snapshotcsn = exrto_snapshot->snapshot_csn; + snapshot->xmin = exrto_snapshot->xmin; + snapshot->xmax = exrto_snapshot->xmax; + snapshot->read_lsn = exrto_snapshot->read_lsn; + + t_thrd.pgxact->xmin = snapshot->xmin; + u_sess->utils_cxt.TransactionXmin = snapshot->xmin; + + t_thrd.proc->exrto_read_lsn = snapshot->read_lsn; + t_thrd.proc->exrto_min = snapshot->read_lsn; + LWLockRelease(ExrtoSnapshotLock); + + if (t_thrd.proc->exrto_gen_snap_time == 0) { + t_thrd.proc->exrto_gen_snap_time = GetCurrentTimestamp(); + } + Assert(XLogRecPtrIsValid(t_thrd.proc->exrto_read_lsn)); +} + +static inline uint64 get_force_recycle_pos(uint64 recycle_pos, uint64 insert_pos) +{ + const double force_recyle_ratio = 0.3; /* to be adjusted */ + Assert(recycle_pos <= insert_pos); + return recycle_pos + (uint64)((insert_pos - recycle_pos) * force_recyle_ratio); +} + +XLogRecPtr calculate_force_recycle_lsn_per_worker(StandbyReadMetaInfo *meta_info) +{ + uint64 base_page_recycle_pos; + uint64 lsn_info_recycle_pos; + XLogRecPtr base_page_recycle_lsn = InvalidXLogRecPtr; + XLogRecPtr lsn_info_recycle_lsn = InvalidXLogRecPtr; + Buffer buffer; + Page page; + + /* for base page */ + if (meta_info->base_page_recyle_position < meta_info->base_page_next_position) { + base_page_recycle_pos = + get_force_recycle_pos(meta_info->base_page_recyle_position, meta_info->base_page_next_position); + buffer = extreme_rto_standby_read::buffer_read_base_page( + meta_info->batch_id, meta_info->redo_id, base_page_recycle_pos, RBM_NORMAL); + LockBuffer(buffer, BUFFER_LOCK_SHARE); + base_page_recycle_lsn = PageGetLSN(BufferGetPage(buffer)); + UnlockReleaseBuffer(buffer); + } + + /* for lsn info */ + if (meta_info->lsn_table_recyle_position < meta_info->lsn_table_next_position) { + lsn_info_recycle_pos = + get_force_recycle_pos(meta_info->lsn_table_recyle_position, meta_info->lsn_table_next_position); + page = extreme_rto_standby_read::get_lsn_info_page( + meta_info->batch_id, meta_info->redo_id, lsn_info_recycle_pos, RBM_NORMAL, &buffer); + if (unlikely(page == NULL || buffer == InvalidBuffer)) { + ereport(PANIC, + (errmsg(EXRTOFORMAT("get_lsn_info_page failed, batch_id: %u, redo_id: %u, pos: %lu"), + meta_info->batch_id, + meta_info->redo_id, + lsn_info_recycle_pos))); + } + LockBuffer(buffer, BUFFER_LOCK_SHARE); + extreme_rto_standby_read::LsnInfo lsn_info = + (extreme_rto_standby_read::LsnInfo)(page + extreme_rto_standby_read::LSN_INFO_HEAD_SIZE); + lsn_info_recycle_lsn = lsn_info->lsn[0]; + UnlockReleaseBuffer(buffer); + } + + return rtl::max(base_page_recycle_lsn, lsn_info_recycle_lsn); +} + +void calculate_force_recycle_lsn(XLogRecPtr &recycle_lsn) +{ + XLogRecPtr recycle_lsn_per_worker; + uint32 worker_nums = extreme_rto::g_dispatcher->allWorkersCnt; + extreme_rto::PageRedoWorker **workers = extreme_rto::g_dispatcher->allWorkers; + + for (uint32 i = 0; i < worker_nums; ++i) { + extreme_rto::PageRedoWorker *page_redo_worker = workers[i]; + if (page_redo_worker->role != extreme_rto::REDO_PAGE_WORKER || (page_redo_worker->isUndoSpaceWorker)) { + continue; + } + recycle_lsn_per_worker = calculate_force_recycle_lsn_per_worker(&page_redo_worker->standby_read_meta_info); + if (XLByteLT(recycle_lsn, recycle_lsn_per_worker)) { + recycle_lsn = recycle_lsn_per_worker; + } + } + ereport(LOG, + (errmsg(EXRTOFORMAT("[exrto_recycle] try force recycle, recycle lsn: %08X/%08X"), + (uint32)(recycle_lsn >> UINT64_HALF), + (uint32)recycle_lsn))); +} + +static inline bool exceed_standby_max_query_time(TimestampTz start_time) +{ + if (start_time == 0) { + return false; + } + return TimestampDifferenceExceeds( + start_time, GetCurrentTimestamp(), g_instance.attr.attr_storage.standby_max_query_time * MSECS_PER_SEC); +} + +/* 1. resolve recycle conflict with backends + * 2. get oldest xmin and oldest readlsn of backends. */ +void proc_array_get_oldeset_readlsn( + XLogRecPtr recycle_lsn, XLogRecPtr &oldest_lsn, TransactionId &oldest_xmin, bool &conflict) +{ + ProcArrayStruct *proc_array = g_instance.proc_array_idx; + conflict = false; + + (void)LWLockAcquire(ProcArrayLock, LW_SHARED); + for (int index = 0; index < proc_array->numProcs; index++) { + int pg_proc_no = proc_array->pgprocnos[index]; + PGPROC *pg_proc = g_instance.proc_base_all_procs[pg_proc_no]; + PGXACT *pg_xact = &g_instance.proc_base_all_xacts[pg_proc_no]; + TransactionId pxmin = pg_xact->xmin; + XLogRecPtr read_lsn = pg_proc->exrto_min; + ereport(DEBUG1, + (errmsg(EXRTOFORMAT("proc_array_get_oldeset_readlsn info, read_lsn: %08X/%08X ,xmin: %lu ,vacuum_flags: " + "%hhu ,pid: %lu"), + (uint32)(read_lsn >> UINT64_HALF), + (uint32)read_lsn, + pxmin, + pg_xact->vacuumFlags, + pg_proc->pid))); + + if (pg_proc->pid == 0 || XLogRecPtrIsInvalid(read_lsn)) { + continue; + } + + Assert(!(pg_xact->vacuumFlags & PROC_IN_VACUUM)); + /* + * Backend is doing logical decoding which manages xmin + * separately, check below. + */ + if (pg_xact->vacuumFlags & PROC_IN_LOGICAL_DECODING) { + continue; + } + + /* cancel query when its read_lsn < recycle_lsn or its runtime > standby_max_query_time */ + if (XLByteLT(read_lsn, recycle_lsn) || exceed_standby_max_query_time(pg_proc->exrto_gen_snap_time)) { + pg_proc->recoveryConflictPending = true; + conflict = true; + if (pg_proc->pid != 0) { + /* + * Kill the pid if it's still here. If not, that's what we + * wanted so ignore any errors. + */ + (void)SendProcSignal(pg_proc->pid, PROCSIG_RECOVERY_CONFLICT_SNAPSHOT, pg_proc->backendId); + ereport(LOG, + (errmsg( + EXRTOFORMAT("read_lsn is less than recycle_lsn or query time exceed max_query_time while " + "get_oldeset_readlsn, read_lsn %lu, " + "recycle_lsn: %lu, exrto_gen_snap_time: %ld, current_time: %ld, thread id = %lu\n"), + read_lsn, + recycle_lsn, + pg_proc->exrto_gen_snap_time, + GetCurrentTimestamp(), + pg_proc->pid))); + /* + * Wait a little bit for it to die so that we avoid flooding + * an unresponsive backend when system is heavily loaded. + */ + pg_usleep(5000L); + } + continue; + } + + if (XLogRecPtrIsInvalid(oldest_lsn) || (XLogRecPtrIsValid(read_lsn) && XLByteLT(read_lsn, oldest_lsn))) { + oldest_lsn = read_lsn; + } + + if (!TransactionIdIsValid(oldest_xmin) || + (TransactionIdIsValid(pxmin) && TransactionIdFollows(oldest_xmin, pxmin))) { + oldest_xmin = pxmin; + } + } + LWLockRelease(ProcArrayLock); +} + +void proc_array_get_oldeset_xmin_for_undo(TransactionId &oldest_xmin) +{ + ProcArrayStruct *proc_array = g_instance.proc_array_idx; + + (void)LWLockAcquire(ProcArrayLock, LW_SHARED); + for (int index = 0; index < proc_array->numProcs; index++) { + int pg_proc_no = proc_array->pgprocnos[index]; + PGPROC *pg_proc = g_instance.proc_base_all_procs[pg_proc_no]; + PGXACT *pg_xact = &g_instance.proc_base_all_xacts[pg_proc_no]; + TransactionId pxmin = pg_xact->xmin; + + if (pg_proc->pid == 0 || !TransactionIdIsValid(pxmin)) { + continue; + } + + Assert(!(pg_xact->vacuumFlags & PROC_IN_VACUUM)); + /* + * Backend is doing logical decoding which manages xmin + * separately, check below. + */ + if (pg_xact->vacuumFlags & PROC_IN_LOGICAL_DECODING) { + continue; + } + if (!TransactionIdIsValid(oldest_xmin) || + (TransactionIdIsValid(pxmin) && TransactionIdFollows(oldest_xmin, pxmin))) { + oldest_xmin = pxmin; + } + } + LWLockRelease(ProcArrayLock); +} + +XLogRecPtr exrto_calculate_recycle_position(bool force_recyle) +{ + Assert(t_thrd.role != PAGEREDO); + Assert(IS_EXRTO_READ); + + XLogRecPtr recycle_lsn = pg_atomic_read_u64(&g_instance.comm_cxt.predo_cxt.global_recycle_lsn); + XLogRecPtr oldest_lsn = InvalidXLogRecPtr; + TransactionId oldest_xmin = InvalidTransactionId; + bool conflict = false; + const int max_check_times = 1000; + int check_times = 0; + + if (force_recyle) { + calculate_force_recycle_lsn(recycle_lsn); + } + ereport(DEBUG1, + (errmsg(EXRTOFORMAT("time information of calculate recycle position, current_time: %ld, snapshot " + "read_lsn: %08X/%08X, gen_snaptime:%ld"), + GetCurrentTimestamp(), + (uint32)(g_instance.comm_cxt.predo_cxt.exrto_snapshot->read_lsn >> UINT64_HALF), + (uint32)g_instance.comm_cxt.predo_cxt.exrto_snapshot->read_lsn, + g_instance.comm_cxt.predo_cxt.exrto_snapshot->gen_snap_time))); + + /* + * If there is no backend read threads, set read oldest lsn to snapshot lsn. + */ + ExrtoSnapshot exrto_snapshot = NULL; + exrto_snapshot = g_instance.comm_cxt.predo_cxt.exrto_snapshot; + (void)LWLockAcquire(ExrtoSnapshotLock, LW_SHARED); + if (XLByteEQ(exrto_snapshot->read_lsn, 0)) { + ereport(WARNING, (errmsg("could not get a valid snapshot with extreme rto"))); + } else { + oldest_lsn = exrto_snapshot->read_lsn; + oldest_xmin = exrto_snapshot->xmin; + } + LWLockRelease(ExrtoSnapshotLock); + /* Loop checks to avoid conflicting queries that were not successfully canceled. */ + do { + RedoInterruptCallBack(); + proc_array_get_oldeset_readlsn(recycle_lsn, oldest_lsn, oldest_xmin, conflict); + check_times++; + } while (conflict && check_times < max_check_times); + + recycle_lsn = rtl::max(recycle_lsn, oldest_lsn); + + ereport(LOG, + (errmsg( + EXRTOFORMAT( + "[exrto_recycle] calculate recycle position, oldestlsn: %08X/%08X, snapshot read_lsn: %08X/%08X, try " + "recycle lsn: %08X/%08X, xmin: %lu"), + (uint32)(oldest_lsn >> UINT64_HALF), + (uint32)oldest_lsn, + (uint32)(g_instance.comm_cxt.predo_cxt.exrto_snapshot->read_lsn >> UINT64_HALF), + (uint32)g_instance.comm_cxt.predo_cxt.exrto_snapshot->read_lsn, + (uint32)(recycle_lsn >> UINT64_HALF), + (uint32)recycle_lsn, oldest_xmin))); + pg_atomic_write_u64(&g_instance.comm_cxt.predo_cxt.exrto_recyle_xmin, oldest_xmin); + return recycle_lsn; +} + +TransactionId exrto_calculate_recycle_xmin_for_undo() +{ + Assert(t_thrd.role != PAGEREDO); + Assert(IS_EXRTO_READ); + TransactionId oldest_xmin = InvalidTransactionId; + TransactionId snapshot_xmin = InvalidTransactionId; + proc_array_get_oldeset_xmin_for_undo(oldest_xmin); + + /* + * If there is no backend read threads, set read oldest lsn to snapshot lsn. + */ + if ((oldest_xmin == InvalidTransactionId) && (extreme_rto::g_dispatcher != NULL)) { + ExrtoSnapshot exrto_snapshot = NULL; + exrto_snapshot = g_instance.comm_cxt.predo_cxt.exrto_snapshot; + (void)LWLockAcquire(ExrtoSnapshotLock, LW_SHARED); + if (XLByteEQ(exrto_snapshot->xmin, InvalidTransactionId)) { + ereport(WARNING, + (errmsg("exrto_calculate_recycle_xmin_for_undo: could not get a valid snapshot in exrto_snapshot"))); + } else { + snapshot_xmin = exrto_snapshot->xmin; + } + + LWLockRelease(ExrtoSnapshotLock); + } + ereport(DEBUG1, + (errmodule(MOD_UNDO), + errmsg(UNDOFORMAT("exrto_calculate_recycle_xmin_for_undo: oldest_xmin: %lu, snapshot_xmin: %lu."), + oldest_xmin, + snapshot_xmin))); + + if (oldest_xmin == InvalidTransactionId) { + return snapshot_xmin; + } + return oldest_xmin; +} diff --git a/src/gausskernel/storage/access/rmgrdesc/standbydesc.cpp b/src/gausskernel/storage/access/rmgrdesc/standbydesc.cpp index 50197b56f..76adbc238 100644 --- a/src/gausskernel/storage/access/rmgrdesc/standbydesc.cpp +++ b/src/gausskernel/storage/access/rmgrdesc/standbydesc.cpp @@ -45,36 +45,47 @@ void standby_desc(StringInfo buf, XLogReaderState *record) char *rec = XLogRecGetData(record); uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; if (info == XLOG_STANDBY_LOCK) { - xl_standby_locks *xlrec = (xl_standby_locks *)rec; - int i; - - appendStringInfo(buf, "AccessExclusive locks: nlocks %d ", xlrec->nlocks); - - for (i = 0; i < xlrec->nlocks; i++) - appendStringInfo(buf, " xid " XID_FMT " db %u rel %u", xlrec->locks[i].xid, xlrec->locks[i].dbOid, - xlrec->locks[i].relOid); + if ((XLogRecGetInfo(record) & PARTITION_ACCESS_EXCLUSIVE_LOCK_UPGRADE_FLAG) == 0) { + xl_standby_locks *xlrec = (xl_standby_locks *)rec; + appendStringInfo(buf, "AccessExclusive locks: nlocks %d ", xlrec->nlocks); + for (int i = 0; i < xlrec->nlocks; i++) { + appendStringInfo(buf, " xid " XID_FMT " db %u rel %u seq %u", xlrec->locks[i].xid, + xlrec->locks[i].dbOid, xlrec->locks[i].relOid, InvalidOid); + } + } else { + XLogStandbyLocksNew *xlrec = (XLogStandbyLocksNew *)rec; + appendStringInfo(buf, "AccessExclusive locks: nlocks %d ", xlrec->nlocks); + for (int i = 0; i < xlrec->nlocks; i++) { + appendStringInfo(buf, " xid " XID_FMT " db %u rel %u seq %u", xlrec->locks[i].xid, + xlrec->locks[i].dbOid, xlrec->locks[i].relOid, xlrec->locks[i].seq); + } + } } else if (info == XLOG_RUNNING_XACTS) { appendStringInfo(buf, " XLOG_RUNNING_XACTS"); } else if (info == XLOG_STANDBY_CSN) { appendStringInfo(buf, " XLOG_STANDBY_CSN"); } else if (info == XLOG_STANDBY_UNLOCK) { - xl_standby_locks *xlrec = (xl_standby_locks *)rec; - int i; - - appendStringInfo(buf, "release AccessExclusive locks: nlocks %d ", xlrec->nlocks); - - for (i = 0; i < xlrec->nlocks; i++) { - appendStringInfo(buf, " xid " XID_FMT " db %u rel %u", xlrec->locks[i].xid, xlrec->locks[i].dbOid, - xlrec->locks[i].relOid); + if ((XLogRecGetInfo(record) & PARTITION_ACCESS_EXCLUSIVE_LOCK_UPGRADE_FLAG) == 0) { + xl_standby_locks *xlrec = (xl_standby_locks *)rec; + appendStringInfo(buf, "AccessExclusive locks: nlocks %d ", xlrec->nlocks); + for (int i = 0; i < xlrec->nlocks; i++) { + appendStringInfo(buf, " xid " XID_FMT " db %u rel %u seq %u", xlrec->locks[i].xid, + xlrec->locks[i].dbOid, xlrec->locks[i].relOid, InvalidOid); + } + } else { + XLogStandbyLocksNew *xlrec = (XLogStandbyLocksNew *)rec; + appendStringInfo(buf, "AccessExclusive locks: nlocks %d ", xlrec->nlocks); + for (int i = 0; i < xlrec->nlocks; i++) { + appendStringInfo(buf, " xid " XID_FMT " db %u rel %u seq %u", xlrec->locks[i].xid, + xlrec->locks[i].dbOid, xlrec->locks[i].relOid, xlrec->locks[i].seq); + } } - } else if (info == XLOG_STANDBY_CSN_COMMITTING) { - uint64* id = ((uint64 *)XLogRecGetData(record)); + uint64 *id = ((uint64 *)XLogRecGetData(record)); appendStringInfo(buf, " XLOG_STANDBY_CSN_COMMITTING, xid %lu, csn %lu", id[0], id[1]); } else if (info == XLOG_STANDBY_CSN_ABORTED) { - uint64* id = ((uint64 *)XLogRecGetData(record)); + uint64 *id = ((uint64 *)XLogRecGetData(record)); appendStringInfo(buf, " XLOG_STANDBY_CSN_ABORTED, xid %lu", id[0]); - } else appendStringInfo(buf, "UNKNOWN"); } diff --git a/src/gausskernel/storage/access/transam/clog.cpp b/src/gausskernel/storage/access/transam/clog.cpp index 0decff72f..4109ef785 100644 --- a/src/gausskernel/storage/access/transam/clog.cpp +++ b/src/gausskernel/storage/access/transam/clog.cpp @@ -42,12 +42,15 @@ #include "access/xlog.h" #include "access/xloginsert.h" #include "access/xlogutils.h" +#include "access/extreme_rto/standby_read/standby_read_delay_ddl.h" +#include "access/multi_redo_api.h" #include "miscadmin.h" #include "pgstat.h" #include "pg_trace.h" #include "storage/smgr/fd.h" #include "storage/proc.h" #include "storage/file/fio_device.h" +#include "storage/procarray.h" #ifdef USE_ASSERT_CHECKING #include "utils/builtins.h" #endif /* USE_ASSERT_CHECKING */ @@ -1105,6 +1108,20 @@ static void WriteTruncateXlogRec(int64 pageno) XLogWaitFlush(recptr); } +void clog_redo_truncate_cancel_conflicting_proc(TransactionId latest_removed_xid, XLogRecPtr lsn) +{ + const int max_check_times = 1000; + int check_times = 0; + bool conflict = true; + bool reach_max_check_times = false; + while (conflict && check_times < max_check_times) { + RedoInterruptCallBack(); + check_times++; + reach_max_check_times = (check_times == max_check_times); + conflict = proc_array_cancel_conflicting_proc(latest_removed_xid, lsn, reach_max_check_times); + } +} + /* * CLOG resource manager's routines */ @@ -1137,12 +1154,22 @@ void clog_redo(XLogReaderState *record) rc = memcpy_s(&pageno, sizeof(int64), XLogRecGetData(record), sizeof(int64)); securec_check(rc, "", ""); + if (IS_EXRTO_READ) { + update_delay_ddl_file_truncate_clog(record->ReadRecPtr, pageno); + return; + } /* * During XLOG replay, latest_page_number isn't set up yet; insert a * suitable value to bypass the sanity test in SimpleLruTruncate. */ ClogCtl(pageno)->shared->latest_page_number = pageno; + TransactionId truncate_xid = (TransactionId)PAGE_TO_TRANSACTION_ID(pageno); + clog_redo_truncate_cancel_conflicting_proc(truncate_xid, InvalidXLogRecPtr); + if (TransactionIdPrecedes(g_instance.undo_cxt.hotStandbyRecycleXid, truncate_xid)) { + pg_atomic_write_u64(&g_instance.undo_cxt.hotStandbyRecycleXid, truncate_xid); + } + SimpleLruTruncate(ClogCtl(0), pageno, NUM_CLOG_PARTITIONS); DeleteObsoleteTwoPhaseFile(pageno); } else @@ -1302,3 +1329,4 @@ void SSCLOGShmemClear(void) CBufMappingPartitionLockByIndex(i), CLOGDIR); } } + diff --git a/src/gausskernel/storage/access/transam/extreme_rto/dispatcher.cpp b/src/gausskernel/storage/access/transam/extreme_rto/dispatcher.cpp index d24d2fdbd..eb77095e0 100755 --- a/src/gausskernel/storage/access/transam/extreme_rto/dispatcher.cpp +++ b/src/gausskernel/storage/access/transam/extreme_rto/dispatcher.cpp @@ -221,7 +221,7 @@ static const RmgrDispatchData g_dispatchTable[RM_MAX_ID + 1] = { }; const int REDO_WAIT_SLEEP_TIME = 5000; /* 5ms */ -const int MAX_REDO_WAIT_LOOP = 24000; /* 5ms*24000 = 2min */ +const int MAX_REDO_WAIT_LOOP = 48000; /* 5ms*24000 = 2min */ uint32 GetReadyWorker() { @@ -1320,7 +1320,8 @@ static bool DispatchCompresseShrinkRecord(XLogReaderState *record, List *expecte static bool DispatchBtreeRecord(XLogReaderState *record, List *expectedTLIs, TimestampTz recordXTime) { uint8 info = (XLogRecGetInfo(record) & (~XLR_INFO_MASK)); - if (info == XLOG_BTREE_REUSE_PAGE) { + if (info == XLOG_BTREE_REUSE_PAGE && + !(IS_EXRTO_STANDBY_READ && g_instance.attr.attr_storage.enable_exrto_standby_read_opt)) { DispatchTxnRecord(record, expectedTLIs); } else { DispatchRecordWithPages(record, expectedTLIs); @@ -1332,7 +1333,8 @@ static bool DispatchBtreeRecord(XLogReaderState *record, List *expectedTLIs, Tim static bool DispatchUBTreeRecord(XLogReaderState *record, List *expectedTLIs, TimestampTz recordXTime) { uint8 info = (XLogRecGetInfo(record) & (~XLR_INFO_MASK)); - if (info == XLOG_UBTREE_REUSE_PAGE) { + if (info == XLOG_UBTREE_REUSE_PAGE && + !(IS_EXRTO_STANDBY_READ && g_instance.attr.attr_storage.enable_exrto_standby_read_opt)) { DispatchTxnRecord(record, expectedTLIs); } else { DispatchRecordWithPages(record, expectedTLIs); @@ -1435,7 +1437,11 @@ static bool DispatchHeap2VacuumRecord(XLogReaderState *record, List *expectedTLI uint8 info = ((XLogRecGetInfo(record) & (~XLR_INFO_MASK)) & XLOG_HEAP_OPMASK); if (info == XLOG_HEAP2_CLEANUP_INFO) { - DispatchTxnRecord(record, expectedTLIs); + xl_heap_cleanup_info *xlrec = (xl_heap_cleanup_info *)XLogRecGetData(record); + RelFileNode tmp_node; + RelFileNodeCopy(tmp_node, xlrec->node, (int2)XLogRecGetBucketId(record)); + + DispatchToOnePageWorker(record, tmp_node, expectedTLIs); } else { DispatchRecordWithPages(record, expectedTLIs); } diff --git a/src/gausskernel/storage/access/transam/extreme_rto/exrto_recycle.cpp b/src/gausskernel/storage/access/transam/extreme_rto/exrto_recycle.cpp index 18c47d565..a7d1389eb 100644 --- a/src/gausskernel/storage/access/transam/extreme_rto/exrto_recycle.cpp +++ b/src/gausskernel/storage/access/transam/extreme_rto/exrto_recycle.cpp @@ -122,8 +122,11 @@ bool check_if_need_force_recycle() total_lsn_info_size += (meta_info.lsn_table_next_position - meta_info.lsn_table_recyle_position); } - if (total_base_page_size > g_instance.attr.attr_storage.max_standby_base_page_size * ratio || - total_lsn_info_size > g_instance.attr.attr_storage.max_standby_lsn_info_size * ratio) { + /* the unit of max_standby_base_page_size and max_standby_lsn_info_size is KB */ + uint64 max_standby_base_page_size = ((uint64)u_sess->attr.attr_storage.max_standby_base_page_size << 10); + uint64 max_standby_lsn_info_size = ((uint64)u_sess->attr.attr_storage.max_standby_lsn_info_size << 10); + if (total_base_page_size > max_standby_base_page_size * ratio || + total_lsn_info_size > max_standby_lsn_info_size * ratio) { return true; } @@ -145,6 +148,7 @@ void do_standby_read_recyle(XLogRecPtr recycle_lsn) XLByteLT(page_redo_worker->standby_read_meta_info.recycle_lsn_per_worker, min_recycle_lsn)) { min_recycle_lsn = page_redo_worker->standby_read_meta_info.recycle_lsn_per_worker; } + pg_usleep(1000); // sleep 1ms } if (XLByteLT(g_instance.comm_cxt.predo_cxt.global_recycle_lsn, min_recycle_lsn)) { pg_atomic_write_u64(&g_instance.comm_cxt.predo_cxt.global_recycle_lsn, min_recycle_lsn); diff --git a/src/gausskernel/storage/access/transam/extreme_rto/page_redo.cpp b/src/gausskernel/storage/access/transam/extreme_rto/page_redo.cpp index 7d5a19da6..1b2db5d13 100755 --- a/src/gausskernel/storage/access/transam/extreme_rto/page_redo.cpp +++ b/src/gausskernel/storage/access/transam/extreme_rto/page_redo.cpp @@ -100,7 +100,7 @@ namespace extreme_rto { static const int MAX_PARSE_BUFF_NUM = PAGE_WORK_QUEUE_SIZE * 10 * 3; static const int MAX_LOCAL_BUFF_NUM = PAGE_WORK_QUEUE_SIZE * 10 * 3; -static const int MAX_CLEAR_SMGR_NUM = 100000; +static const int MAX_CLEAR_SMGR_NUM = 50000; static const char *const PROCESS_TYPE_CMD_ARG = "--forkpageredo"; static char g_AUXILIARY_TYPE_CMD_ARG[16] = {0}; @@ -117,6 +117,7 @@ RedoItem g_cleanInvalidPageMark; static const int PAGE_REDO_WORKER_ARG = 3; static const int REDO_SLEEP_50US = 50; static const int REDO_SLEEP_100US = 100; +static const int EXRTO_STANDBY_READ_TIME_INTERVAL = 1 * 1000; static void ApplySinglePageRecord(RedoItem *); static void InitGlobals(); @@ -342,6 +343,23 @@ uint32 GetMyPageRedoWorkerIdWithLock() return g_redoWorker->id; } +void redo_worker_release_all_locks() +{ + Assert(t_thrd.proc != NULL); + + /* If waiting, get off wait queue (should only be needed after error) */ + LockErrorCleanup(); + + /* Release standard locks, including session-level if aborting */ + LockReleaseAll(DEFAULT_LOCKMETHOD, true); + + /* + * User locks are not released by transaction end, so be sure to release + * them explicitly. + */ + LockReleaseAll(USER_LOCKMETHOD, true); +} + /* Run from any worker thread. */ PGPROC *GetPageRedoWorkerProc(PageRedoWorker *worker) { @@ -393,6 +411,24 @@ void HandlePageRedoInterrupts() HandlePageRedoInterruptsImpl(); } +void clean_smgr(uint64 &clear_redo_fd_count) +{ + const uint64 clear_redo_fd_count_mask = 0x3FFFFF; + clear_redo_fd_count += 1; + if (clear_redo_fd_count > clear_redo_fd_count_mask && GetSMgrRelationHash() != NULL) { + clear_redo_fd_count = 0; + long hash_num = hash_get_num_entries(GetSMgrRelationHash()); + if (hash_num >= MAX_CLEAR_SMGR_NUM) { + ereport(LOG, + (errmsg("smgr close all: clear_redo_fd_count:%lu, hash_num:%ld,clear_redo_fd_count_mask :%lu", + clear_redo_fd_count, + hash_num, + clear_redo_fd_count_mask))); + smgrcloseall(); + } + } +} + void ReferenceRedoItem(void *item) { RedoItem *redoItem = (RedoItem *)item; @@ -444,7 +480,6 @@ void AddRecordReadBlocks(void *rec, uint32 readblocks) void AddRefRecord(void *rec) { - pg_memory_barrier(); #ifndef EXTREME_RTO_DEBUG (void)pg_atomic_fetch_add_u32(&((XLogReaderState *)rec)->refcount, 1); #else @@ -478,7 +513,6 @@ void AddRefRecord(void *rec) void SubRefRecord(void *rec) { - pg_memory_barrier(); Assert(((XLogReaderState *)rec)->refcount != 0); uint32 relCount = pg_atomic_sub_fetch_u32(&((XLogReaderState *)rec)->refcount, 1); #ifdef EXTREME_RTO_DEBUG @@ -961,6 +995,27 @@ static void WaitNextBarrier(XLogRecParseState *parseState) } } +void redo_page_manager_do_cleanup_action(XLogRecParseState *parse_state) +{ + if (!IS_EXRTO_READ_OPT || !pm_state_is_hot_standby()) { + return; + } + + RelFileNode tmp_node; + tmp_node.spcNode = parse_state->blockparse.blockhead.spcNode; + tmp_node.dbNode = parse_state->blockparse.blockhead.dbNode; + tmp_node.relNode = parse_state->blockparse.blockhead.relNode; + tmp_node.bucketNode = parse_state->blockparse.blockhead.bucketNode; + tmp_node.opt = parse_state->blockparse.blockhead.opt; + XLogRecPtr lsn = parse_state->blockparse.blockhead.end_ptr; + TransactionId removed_xid = parse_state->blockparse.extra_rec.clean_up_info.removed_xid; + + LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE); + UpdateCleanUpInfo(removed_xid, lsn); + LWLockRelease(ProcArrayLock); + ResolveRecoveryConflictWithSnapshot(removed_xid, tmp_node, lsn); +} + void PageManagerRedoParseState(XLogRecParseState *preState) { switch (preState->blockparse.blockhead.block_valid) { @@ -1033,6 +1088,10 @@ void PageManagerRedoParseState(XLogRecParseState *preState) RedoPageManagerDistributeToAllOneBlock(preState); XLogBlockParseStateRelease(preState); break; + case BLOCK_DATA_CLEANUP_TYPE: + redo_page_manager_do_cleanup_action(preState); + XLogBlockParseStateRelease(preState); + break; default: XLogBlockParseStateRelease(preState); break; @@ -1077,6 +1136,7 @@ bool PageManagerRedoDistributeItems(XLogRecParseState *record_block_state) void RedoPageManagerMain() { XLogRecParseState *record_block_state = NULL; + uint64 clear_redo_fd_count = 0; (void)RegisterRedoInterruptCallBack(HandlePageRedoInterrupts); XLogParseBufferInitFunc(&(g_redoWorker->parseManager), MAX_PARSE_BUFF_NUM, &recordRefOperate, @@ -1097,6 +1157,7 @@ void RedoPageManagerMain() SPSCBlockingQueuePop(g_redoWorker->queue); CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_2]); RedoInterruptCallBack(); + clean_smgr(clear_redo_fd_count); ADD_ABNORMAL_POSITION(5); GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_1]); @@ -1467,6 +1528,7 @@ void RedoPageWorkerMain() RedoInterruptCallBack); } + uint64 clear_redo_fd_count = 0; XLogRecParseState *redoblockstateHead = NULL; GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_1]); while ((redoblockstateHead = (XLogRecParseState *)SPSCBlockingQueueTop(g_redoWorker->queue)) != @@ -1594,6 +1656,7 @@ void RedoPageWorkerMain() } CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_8]); RedoInterruptCallBack(); + clean_smgr(clear_redo_fd_count); CountAndGetRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_2], g_redoWorker->timeCostList[TIME_COST_STEP_1]); ADD_ABNORMAL_POSITION(4); } @@ -1660,8 +1723,14 @@ void PushToWorkerLsn(bool force) cur_recor_count = 0; SendLsnFowarder(); } else { - if (cur_recor_count < max_record_count) { - return; + if (g_instance.attr.attr_storage.EnableHotStandby && pm_state_is_hot_standby()) { + if (!exceed_send_lsn_forworder_interval()) { + return; + } + } else { + if (cur_recor_count < max_record_count) { + return; + } } if (pg_atomic_read_u32(&g_GlobalLsnForwarder.record.refcount) != 0) { @@ -2592,6 +2661,9 @@ void ParallelRedoThreadMain() int retCode = RedoMainLoop(); StandbyReleaseAllLocks(); + if (g_redoWorker->role == REDO_TRXN_WORKER) { + redo_worker_release_all_locks(); + } ResourceManagerStop(); ereport(LOG, (errmsg("Page-redo-worker thread %u terminated, role:%u, slotId:%u, retcode %u.", g_redoWorker->id, g_redoWorker->role, g_redoWorker->slotId, retCode))); @@ -3100,349 +3172,18 @@ void SeqCheckRemoteReadAndRepairPage() } } -inline void invalid_msg_leak_warning(XLogRecPtr trxn_lsn) +bool exceed_send_lsn_forworder_interval() { - if (t_thrd.page_redo_cxt.invalid_msg.valid) { - ereport(WARNING, (errmsg(EXRTOFORMAT("[exrto_generate_snapshot] not send invalid msg: %08X/%08X"), - (uint32)(trxn_lsn >> UINT64_HALF), (uint32)trxn_lsn))); - } -} - -void exrto_generate_snapshot(XLogRecPtr trxn_lsn) -{ - if (!g_instance.attr.attr_storage.EnableHotStandby) { - return; - } - - ExrtoSnapshot exrto_snapshot = g_instance.comm_cxt.predo_cxt.exrto_snapshot; - /* - * do not generate the same snapshot repeatedly. - */ - if (XLByteLE(trxn_lsn, exrto_snapshot->read_lsn)) { - invalid_msg_leak_warning(trxn_lsn); - return; - } + TimestampTz last_time; + TimestampTz now_time; - TransactionId xmin; - TransactionId xmax; - CommitSeqNo snapshot_csn; - - exrto_get_snapshot_data(xmin, xmax, snapshot_csn); - (void)LWLockAcquire(ExrtoSnapshotLock, LW_EXCLUSIVE); - exrto_snapshot->snapshot_csn = snapshot_csn; - exrto_snapshot->xmin = xmin; - exrto_snapshot->xmax = xmax; - exrto_snapshot->read_lsn = trxn_lsn; - send_delay_invalid_message(); - LWLockRelease(ExrtoSnapshotLock); -} - -void exrto_read_snapshot(Snapshot snapshot) -{ - if ((!is_exrto_standby_read_worker()) || u_sess->proc_cxt.clientIsCMAgent || dummyStandbyMode) { - return; - } - - ExrtoSnapshot exrto_snapshot = g_instance.comm_cxt.predo_cxt.exrto_snapshot; - (void)LWLockAcquire(ExrtoSnapshotLock, LW_SHARED); - if (XLByteEQ(exrto_snapshot->read_lsn, 0)) { - LWLockRelease(ExrtoSnapshotLock); - ereport(ERROR, (errmsg("could not get a valid snapshot with extreme rto"))); - } - snapshot->snapshotcsn = exrto_snapshot->snapshot_csn; - snapshot->xmin = exrto_snapshot->xmin; - snapshot->xmax = exrto_snapshot->xmax; - snapshot->read_lsn = exrto_snapshot->read_lsn; - LWLockRelease(ExrtoSnapshotLock); - if (!TransactionIdIsValid(t_thrd.pgxact->xmin) || TransactionIdPrecedes(snapshot->xmin, t_thrd.pgxact->xmin)) { - t_thrd.pgxact->xmin = snapshot->xmin; - u_sess->utils_cxt.TransactionXmin = snapshot->xmin; - } - t_thrd.proc->exrto_read_lsn = exrto_snapshot->read_lsn; - if (t_thrd.proc->exrto_min == 0 || - XLByteLT(t_thrd.proc->exrto_min, t_thrd.proc->exrto_read_lsn)) { - t_thrd.proc->exrto_min = t_thrd.proc->exrto_read_lsn; - } - - if (t_thrd.proc->exrto_gen_snap_time == 0) { - t_thrd.proc->exrto_gen_snap_time = GetCurrentTimestamp(); - } - Assert(XLogRecPtrIsValid(t_thrd.proc->exrto_read_lsn)); -} - -static inline uint64 get_force_recycle_pos(uint64 recycle_pos, uint64 insert_pos) -{ - const double force_recyle_ratio = 0.3; /* to be adjusted */ - Assert(recycle_pos <= insert_pos); - return recycle_pos + (uint64)((insert_pos - recycle_pos) * force_recyle_ratio); -} - -XLogRecPtr calculate_force_recycle_lsn_per_worker(StandbyReadMetaInfo* meta_info) -{ - uint64 base_page_recycle_pos; - uint64 lsn_info_recycle_pos; - XLogRecPtr base_page_recycle_lsn = InvalidXLogRecPtr; - XLogRecPtr lsn_info_recycle_lsn = InvalidXLogRecPtr; - Buffer buffer; - Page page; - - /* for base page */ - if (meta_info->base_page_recyle_position < meta_info->base_page_next_position) { - base_page_recycle_pos = get_force_recycle_pos(meta_info->base_page_recyle_position, - meta_info->base_page_next_position); - buffer = extreme_rto_standby_read::buffer_read_base_page(meta_info->batch_id, meta_info->redo_id, - base_page_recycle_pos, RBM_NORMAL); - LockBuffer(buffer, BUFFER_LOCK_SHARE); - base_page_recycle_lsn = PageGetLSN(BufferGetPage(buffer)); - UnlockReleaseBuffer(buffer); - } - - /* for lsn info */ - if (meta_info->lsn_table_recyle_position < meta_info->lsn_table_next_position) { - lsn_info_recycle_pos = get_force_recycle_pos(meta_info->lsn_table_recyle_position, - meta_info->lsn_table_next_position); - page = extreme_rto_standby_read::get_lsn_info_page(meta_info->batch_id, meta_info->redo_id, - lsn_info_recycle_pos, RBM_NORMAL, &buffer); - if (unlikely(page == NULL || buffer == InvalidBuffer)) { - ereport(PANIC, - (errmsg(EXRTOFORMAT("get_lsn_info_page failed, batch_id: %u, redo_id: %u, pos: %lu"), - meta_info->batch_id, meta_info->redo_id, lsn_info_recycle_pos))); - } - LockBuffer(buffer, BUFFER_LOCK_SHARE); - extreme_rto_standby_read::LsnInfo lsn_info = - (extreme_rto_standby_read::LsnInfo)(page + extreme_rto_standby_read::LSN_INFO_HEAD_SIZE); - lsn_info_recycle_lsn = lsn_info->lsn[0]; - UnlockReleaseBuffer(buffer); - } - - return rtl::max(base_page_recycle_lsn, lsn_info_recycle_lsn); -} - -void calculate_force_recycle_lsn(XLogRecPtr &recycle_lsn) -{ - XLogRecPtr recycle_lsn_per_worker; - uint32 worker_nums = g_dispatcher->allWorkersCnt; - PageRedoWorker** workers = g_dispatcher->allWorkers; - - for (uint32 i = 0; i < worker_nums; ++i) { - PageRedoWorker* page_redo_worker = workers[i]; - if (page_redo_worker->role != REDO_PAGE_WORKER || (page_redo_worker->isUndoSpaceWorker)) { - continue; - } - recycle_lsn_per_worker = calculate_force_recycle_lsn_per_worker(&page_redo_worker->standby_read_meta_info); - if (XLByteLT(recycle_lsn, recycle_lsn_per_worker)) { - recycle_lsn = recycle_lsn_per_worker; - } - } - ereport(LOG, - (errmsg(EXRTOFORMAT("[exrto_recycle] try force recycle, recycle lsn: %08X/%08X"), - (uint32)(recycle_lsn >> UINT64_HALF), (uint32)recycle_lsn))); -} - -static inline bool exceed_standby_max_query_time(TimestampTz start_time) -{ - if (start_time == 0) { + last_time = g_instance.comm_cxt.predo_cxt.exrto_send_lsn_forworder_time; + now_time = GetCurrentTimestamp(); + if (!TimestampDifferenceExceeds(last_time, now_time, EXRTO_STANDBY_READ_TIME_INTERVAL)) { return false; } - return TimestampDifferenceExceeds(start_time, GetCurrentTimestamp(), - g_instance.attr.attr_storage.standby_max_query_time * MSECS_PER_SEC); -} - -/* 1. resolve recycle conflict with backends - * 2. get oldest xmin and oldest readlsn of backends. */ -void proc_array_get_oldeset_readlsn(XLogRecPtr recycle_lsn, XLogRecPtr &oldest_lsn, TransactionId &oldest_xmin, - bool &conflict) -{ - ProcArrayStruct* proc_array = g_instance.proc_array_idx; - conflict = false; - - LWLockAcquire(ProcArrayLock, LW_SHARED); - for (int index = 0; index < proc_array->numProcs; index++) { - int pg_proc_no = proc_array->pgprocnos[index]; - PGPROC* pg_proc = g_instance.proc_base_all_procs[pg_proc_no]; - PGXACT* pg_xact = &g_instance.proc_base_all_xacts[pg_proc_no]; - TransactionId pxmin = pg_xact->xmin; - XLogRecPtr read_lsn = pg_proc->exrto_min; - ereport( - DEBUG1, - (errmsg(EXRTOFORMAT("proc_array_get_oldeset_readlsn info, read_lsn: %08X/%08X ,xmin: %lu ,vacuum_flags: " - "%hhu ,pid: %lu"), - (uint32)(read_lsn >> UINT64_HALF), (uint32)read_lsn, pxmin, pg_xact->vacuumFlags, pg_proc->pid))); - - if (pg_proc->pid == 0 || XLogRecPtrIsInvalid(read_lsn)) { - continue; - } - - Assert(!(pg_xact->vacuumFlags & PROC_IN_VACUUM)); - /* - * Backend is doing logical decoding which manages xmin - * separately, check below. - */ - if (pg_xact->vacuumFlags & PROC_IN_LOGICAL_DECODING) { - continue; - } - - /* cancel query when its read_lsn < recycle_lsn or its runtime > standby_max_query_time */ - if (XLByteLT(read_lsn, recycle_lsn) || exceed_standby_max_query_time(pg_proc->exrto_gen_snap_time)) { - pg_proc->recoveryConflictPending = true; - conflict = true; - if (pg_proc->pid != 0) { - /* - * Kill the pid if it's still here. If not, that's what we - * wanted so ignore any errors. - */ - (void)SendProcSignal(pg_proc->pid, PROCSIG_RECOVERY_CONFLICT_SNAPSHOT, pg_proc->backendId); - ereport( - LOG, - (errmsg( - EXRTOFORMAT("read_lsn is less than recycle_lsn or query time exceed max_query_time while " - "get_oldeset_readlsn, read_lsn %lu, " - "recycle_lsn: %lu, exrto_gen_snap_time: %ld, current_time: %ld, thread id = %lu\n"), - read_lsn, recycle_lsn, pg_proc->exrto_gen_snap_time, GetCurrentTimestamp(), pg_proc->pid))); - /* - * Wait a little bit for it to die so that we avoid flooding - * an unresponsive backend when system is heavily loaded. - */ - pg_usleep(5000L); - } - continue; - } - - if (XLogRecPtrIsInvalid(oldest_lsn) || - (XLogRecPtrIsValid(read_lsn) && XLByteLT(read_lsn, oldest_lsn))) { - oldest_lsn = read_lsn; - } - - if (!TransactionIdIsValid(oldest_xmin) || - (TransactionIdIsValid(pxmin) && TransactionIdFollows(oldest_xmin, pxmin))) { - oldest_xmin = pxmin; - } - } - LWLockRelease(ProcArrayLock); -} - -void proc_array_get_oldeset_xmin_for_undo(TransactionId &oldest_xmin) -{ - ProcArrayStruct* proc_array = g_instance.proc_array_idx; - - LWLockAcquire(ProcArrayLock, LW_SHARED); - for (int index = 0; index < proc_array->numProcs; index++) { - int pg_proc_no = proc_array->pgprocnos[index]; - PGPROC* pg_proc = g_instance.proc_base_all_procs[pg_proc_no]; - PGXACT* pg_xact = &g_instance.proc_base_all_xacts[pg_proc_no]; - TransactionId pxmin = pg_xact->xmin; - - if (pg_proc->pid == 0 || !TransactionIdIsValid(pxmin)) { - continue; - } - - Assert(!(pg_xact->vacuumFlags & PROC_IN_VACUUM)); - /* - * Backend is doing logical decoding which manages xmin - * separately, check below. - */ - if (pg_xact->vacuumFlags & PROC_IN_LOGICAL_DECODING) { - continue; - } - if (!TransactionIdIsValid(oldest_xmin) || - (TransactionIdIsValid(pxmin) && TransactionIdFollows(oldest_xmin, pxmin))) { - oldest_xmin = pxmin; - } - } - LWLockRelease(ProcArrayLock); -} - -XLogRecPtr exrto_calculate_recycle_position(bool force_recyle) -{ - Assert(t_thrd.role != PAGEREDO); - Assert(IS_EXRTO_READ); - - XLogRecPtr recycle_lsn = g_instance.comm_cxt.predo_cxt.global_recycle_lsn; - XLogRecPtr oldest_lsn = InvalidXLogRecPtr; - TransactionId oldest_xmin = InvalidTransactionId; - bool conflict = false; - const int max_check_times = 1000; - int check_times = 0; - - if (force_recyle) { - calculate_force_recycle_lsn(recycle_lsn); - } - ereport(DEBUG1, (errmsg(EXRTOFORMAT("time information of calculate recycle position, current_time: %ld, snapshot " - "read_lsn: %08X/%08X, gen_snaptime:%ld"), - GetCurrentTimestamp(), - (uint32)(g_instance.comm_cxt.predo_cxt.exrto_snapshot->read_lsn >> UINT64_HALF), - (uint32)g_instance.comm_cxt.predo_cxt.exrto_snapshot->read_lsn, - g_instance.comm_cxt.predo_cxt.exrto_snapshot->gen_snap_time))); - - /* - * If there is no backend read threads, set read oldest lsn to snapshot lsn. - */ - ExrtoSnapshot exrto_snapshot = NULL; - exrto_snapshot = g_instance.comm_cxt.predo_cxt.exrto_snapshot; - (void)LWLockAcquire(ExrtoSnapshotLock, LW_SHARED); - if (XLByteEQ(exrto_snapshot->read_lsn, 0)) { - ereport(WARNING, (errmsg("could not get a valid snapshot with extreme rto"))); - } else { - oldest_lsn = exrto_snapshot->read_lsn; - oldest_xmin = exrto_snapshot->xmin; - } - LWLockRelease(ExrtoSnapshotLock); - /* Loop checks to avoid conflicting queries that were not successfully canceled. */ - do { - RedoInterruptCallBack(); - proc_array_get_oldeset_readlsn(recycle_lsn, oldest_lsn, oldest_xmin, conflict); - check_times++; - } while (conflict && check_times < max_check_times); - - recycle_lsn = rtl::max(recycle_lsn, oldest_lsn); - - ereport( - LOG, - (errmsg( - EXRTOFORMAT( - "[exrto_recycle] calculate recycle position, oldestlsn: %08X/%08X, snapshot read_lsn: %08X/%08X, try " - "recycle lsn: %08X/%08X"), - (uint32)(oldest_lsn >> UINT64_HALF), (uint32)oldest_lsn, - (uint32)(g_instance.comm_cxt.predo_cxt.exrto_snapshot->read_lsn >> UINT64_HALF), - (uint32)g_instance.comm_cxt.predo_cxt.exrto_snapshot->read_lsn, - (uint32)(recycle_lsn >> UINT64_HALF), (uint32)recycle_lsn))); - - return recycle_lsn; -} - -TransactionId exrto_calculate_recycle_xmin_for_undo() -{ - Assert(t_thrd.role != PAGEREDO); - Assert(IS_EXRTO_READ); - TransactionId oldest_xmin = InvalidTransactionId; - TransactionId snapshot_xmin = InvalidTransactionId; - proc_array_get_oldeset_xmin_for_undo(oldest_xmin); - - /* - * If there is no backend read threads, set read oldest lsn to snapshot lsn. - */ - if ((oldest_xmin == InvalidTransactionId) && (g_dispatcher != NULL)) { - ExrtoSnapshot exrto_snapshot = NULL; - exrto_snapshot = g_instance.comm_cxt.predo_cxt.exrto_snapshot; - (void)LWLockAcquire(ExrtoSnapshotLock, LW_SHARED); - if (XLByteEQ(exrto_snapshot->xmin, InvalidTransactionId)) { - ereport( - WARNING, - (errmsg("exrto_calculate_recycle_xmin_for_undo: could not get a valid snapshot in exrto_snapshot"))); - } else { - snapshot_xmin = exrto_snapshot->xmin; - } - - LWLockRelease(ExrtoSnapshotLock); - } - ereport(DEBUG1, (errmodule(MOD_UNDO), - errmsg(UNDOFORMAT("exrto_calculate_recycle_xmin_for_undo: oldest_xmin: %lu, snapshot_xmin: %lu."), - oldest_xmin, snapshot_xmin))); - - if (oldest_xmin == InvalidTransactionId) { - return snapshot_xmin; - } - return oldest_xmin; + g_instance.comm_cxt.predo_cxt.exrto_send_lsn_forworder_time = now_time; + return true; } } // namespace extreme_rto \ No newline at end of file diff --git a/src/gausskernel/storage/access/transam/parallel_recovery/dispatcher.cpp b/src/gausskernel/storage/access/transam/parallel_recovery/dispatcher.cpp index b8b574ace..8c1ae4548 100755 --- a/src/gausskernel/storage/access/transam/parallel_recovery/dispatcher.cpp +++ b/src/gausskernel/storage/access/transam/parallel_recovery/dispatcher.cpp @@ -75,6 +75,7 @@ #include "gssignal/gs_signal.h" #include "utils/atomic.h" #include "pgstat.h" +#include "access/xlogreader.h" #ifdef PGXC #include "pgxc/pgxc.h" @@ -327,9 +328,16 @@ void StartRecoveryWorkers(XLogRecPtr startLsn) g_dispatcher = CreateDispatcher(); g_dispatcher->oldCtx = MemoryContextSwitchTo(g_instance.comm_cxt.predo_cxt.parallelRedoCtx); g_dispatcher->txnWorker = StartTxnRedoWorker(); - if (g_dispatcher->txnWorker != NULL) + if (g_dispatcher->txnWorker != NULL) { + Assert(g_instance.comm_cxt.predo_cxt.buffer_pin_wait_buf_len == 0 || + g_instance.comm_cxt.predo_cxt.buffer_pin_wait_buf_len == get_real_recovery_parallelism()); + if (g_instance.comm_cxt.predo_cxt.buffer_pin_wait_buf_ids == NULL) { + g_instance.comm_cxt.predo_cxt.buffer_pin_wait_buf_ids = (int *)MemoryContextAllocZero( + INSTANCE_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_STORAGE), get_real_recovery_parallelism() * sizeof(int)); + g_instance.comm_cxt.predo_cxt.buffer_pin_wait_buf_len = get_real_recovery_parallelism(); + } StartPageRedoWorkers(get_real_recovery_parallelism()); - + } ereport(LOG, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), errmsg("[PR]: max=%d, thrd=%d, workers=%u", g_instance.attr.attr_storage.max_recovery_parallelism, get_real_recovery_parallelism(), g_dispatcher->pageWorkerCount))); @@ -922,10 +930,13 @@ static void DispatchRecordWithPages(XLogReaderState *record, List *expectedTLIs, static bool DispatchHeapRecord(XLogReaderState *record, List *expectedTLIs, TimestampTz recordXTime) { - if (record->max_block_id >= 0) + if (unlikely((XLogRecGetInfo(record) & XLOG_HEAP_OPMASK) == XLOG_HEAP_INPLACE)) { + DispatchRecordWithoutPage(record, expectedTLIs); + } else if (record->max_block_id >= 0) { DispatchRecordWithPages(record, expectedTLIs, SUPPORT_FPAGE_DISPATCH); - else + } else { DispatchRecordWithoutPage(record, expectedTLIs); + } return false; } @@ -1872,7 +1883,7 @@ void SetStartupBufferPinWaitBufId(int bufid) for (uint32 i = 0; i < g_dispatcher->pageWorkerCount; i++) { PGPROC *proc = GetPageRedoWorkerProc(g_dispatcher->pageWorkers[i]); if (t_thrd.proc->pid == proc->pid) { - g_dispatcher->pageWorkers[i]->bufferPinWaitBufId = bufid; + g_instance.comm_cxt.predo_cxt.buffer_pin_wait_buf_ids[i] = bufid; break; } } @@ -1883,7 +1894,7 @@ uint32 GetStartupBufferPinWaitBufLen() { uint32 buf_len = 1; if ((get_real_recovery_parallelism() > 1) && (GetPageWorkerCount() > 0)) { - buf_len += g_dispatcher->pageWorkerCount; + buf_len += g_instance.comm_cxt.predo_cxt.buffer_pin_wait_buf_len; } return buf_len; } @@ -1896,7 +1907,7 @@ void GetStartupBufferPinWaitBufId(int *bufids, uint32 len) { if (g_dispatcher != NULL) { for (uint32 i = 0; i < len - 1; i++) { - bufids[i] = g_dispatcher->pageWorkers[i]->bufferPinWaitBufId; + bufids[i] = g_instance.comm_cxt.predo_cxt.buffer_pin_wait_buf_ids[i]; } bufids[len - 1] = g_instance.proc_base->startupBufferPinWaitBufId; } diff --git a/src/gausskernel/storage/access/transam/parallel_recovery/page_redo.cpp b/src/gausskernel/storage/access/transam/parallel_recovery/page_redo.cpp index f2b17460a..a7981319c 100755 --- a/src/gausskernel/storage/access/transam/parallel_recovery/page_redo.cpp +++ b/src/gausskernel/storage/access/transam/parallel_recovery/page_redo.cpp @@ -190,7 +190,6 @@ static PageRedoWorker *CreateWorker(uint32 id) worker->statWaitReach = 0; worker->statWaitReplay = 0; worker->oldCtx = NULL; - worker->bufferPinWaitBufId = -1; worker->remoteReadPageNum = 0; worker->badPageHashTbl = BadBlockHashTblCreate(); diff --git a/src/gausskernel/storage/access/transam/parallel_recovery/spsc_blocking_queue.cpp b/src/gausskernel/storage/access/transam/parallel_recovery/spsc_blocking_queue.cpp index b63bf9efd..9ccfa760a 100644 --- a/src/gausskernel/storage/access/transam/parallel_recovery/spsc_blocking_queue.cpp +++ b/src/gausskernel/storage/access/transam/parallel_recovery/spsc_blocking_queue.cpp @@ -183,10 +183,9 @@ void *SPSCBlockingQueueTop(SPSCBlockingQueue *queue) else sleeptime = MAX_REDO_QUE_TAKE_DELAY; pg_usleep(sleeptime); - } - - if (queue->callBackFunc != NULL) { - queue->callBackFunc(); + if (queue->callBackFunc != NULL) { + queue->callBackFunc(); + } } } while (COUNT(head, tail, queue->mask) == 0); t_thrd.page_redo_cxt.sleep_long = false; diff --git a/src/gausskernel/storage/access/transam/slru.cpp b/src/gausskernel/storage/access/transam/slru.cpp index 6d433ec07..63e5a94b9 100644 --- a/src/gausskernel/storage/access/transam/slru.cpp +++ b/src/gausskernel/storage/access/transam/slru.cpp @@ -941,8 +941,9 @@ static void SlruReportIOError(SlruCtl ctl, int64 pageno, TransactionId xid) switch (t_thrd.xact_cxt.slru_errcause) { case SLRU_OPEN_FAILED: ereport(ERROR, (errmodule(MOD_SLRU), errcode_for_file_access(), - errmsg("could not access status of transaction %lu , nextXid is %lu ", xid, - t_thrd.xact_cxt.ShmemVariableCache->nextXid), + errmsg("could not access status of transaction %lu , nextXid is %lu, pageno %ld, " + "t_thrd.pgxact->xmin %lu", + xid, t_thrd.xact_cxt.ShmemVariableCache->nextXid, pageno, t_thrd.pgxact->xmin), errdetail("Could not open file \"%s\": %m.", path))); break; case SLRU_SEEK_FAILED: diff --git a/src/gausskernel/storage/access/transam/transam.cpp b/src/gausskernel/storage/access/transam/transam.cpp index 2a1167abe..103888be6 100644 --- a/src/gausskernel/storage/access/transam/transam.cpp +++ b/src/gausskernel/storage/access/transam/transam.cpp @@ -464,7 +464,8 @@ bool UHeapTransactionIdDidCommit(TransactionId transactionId) return true; } if (TransactionIdIsNormal(transactionId) && - TransactionIdPrecedes(transactionId, pg_atomic_read_u64(&g_instance.undo_cxt.globalRecycleXid))) { + TransactionIdPrecedes(transactionId, pg_atomic_read_u64(&g_instance.undo_cxt.globalRecycleXid)) && + !RecoveryInProgress()) { Assert(TransactionIdDidCommit(transactionId)); return true; } diff --git a/src/gausskernel/storage/access/transam/twophase_rmgr.cpp b/src/gausskernel/storage/access/transam/twophase_rmgr.cpp index e0e72b130..9e8702a48 100644 --- a/src/gausskernel/storage/access/transam/twophase_rmgr.cpp +++ b/src/gausskernel/storage/access/transam/twophase_rmgr.cpp @@ -45,11 +45,3 @@ const TwoPhaseCallback g_twophase_postabort_callbacks[TWOPHASE_RM_MAX_ID + 1] = multixact_twophase_postabort, /* MultiXact */ NULL /* PredicateLock */ }; - -const TwoPhaseCallback g_twophase_standby_recover_callbacks[TWOPHASE_RM_MAX_ID + 1] = { - NULL, /* END ID */ - lock_twophase_standby_recover, /* Lock */ - NULL, /* pgstat */ - NULL, /* MultiXact */ - NULL /* PredicateLock */ -}; diff --git a/src/gausskernel/storage/access/transam/xact.cpp b/src/gausskernel/storage/access/transam/xact.cpp index a6ff43cd4..3e21f1bd2 100755 --- a/src/gausskernel/storage/access/transam/xact.cpp +++ b/src/gausskernel/storage/access/transam/xact.cpp @@ -7175,7 +7175,7 @@ void push_unlink_rel_to_hashtbl(ColFileNode *xnodes, int nrels) /* * XLOG support routines */ -void unlink_relfiles(_in_ ColFileNode *xnodes, _in_ int nrels) +void unlink_relfiles(_in_ ColFileNode *xnodes, _in_ int nrels, bool is_old_delay_ddl) { ColMainFileNodesCreate(); @@ -7212,7 +7212,7 @@ void unlink_relfiles(_in_ ColFileNode *xnodes, _in_ int nrels) /* * recycle exrto files when dropping table occurs. */ - if (RecoveryInProgress() && IS_EXRTO_READ) { + if (!is_old_delay_ddl && RecoveryInProgress() && IS_EXRTO_READ) { RelFileNode block_meta_file = relFileNode; block_meta_file.spcNode = EXRTO_BLOCK_INFO_SPACE_OID; extreme_rto_standby_read::remove_one_block_info_file(block_meta_file); diff --git a/src/gausskernel/storage/access/transam/xlog.cpp b/src/gausskernel/storage/access/transam/xlog.cpp index 30d497a05..17efd733e 100755 --- a/src/gausskernel/storage/access/transam/xlog.cpp +++ b/src/gausskernel/storage/access/transam/xlog.cpp @@ -134,6 +134,7 @@ #include #include #include +#include "access/slru.h" #ifdef ENABLE_MOT #include "storage/mot/mot_fdw.h" @@ -8834,6 +8835,69 @@ void init_extreme_rto_standby_read_first_snapshot(const XLogRecPtr checkpoint_lo } } +static bool hex_string_to_int(char* hex_string, uint32* result) +{ + uint32 num = 0; + char* temp = hex_string; + uint32 c = 0; + uint32 index = 0; + + if (NULL == hex_string) { + return false; + } + + while (*temp++ != '\0') { + num++; + } + + while (num--) { + if (hex_string[num] >= 'A' && hex_string[num] <= 'F') { + c = (uint32)((hex_string[num] - 'A') + 10); + } else if (hex_string[num] >= '0' && hex_string[num] <= '9') { + c = (uint32)(hex_string[num] - '0'); + } else { + return false; + } + + *result += c << (index * 4); + index++; + } + + return true; +} + +static inline void set_hot_standby_recycle_xid() +{ + DIR *dir = NULL; + struct dirent *ptr = NULL; + uint32 segnum = 0; + char *dir_name = "pg_clog"; + + if ((dir = opendir(dir_name)) == NULL) { + return; + } + // find the first clog file + while ((ptr = readdir(dir)) != NULL) { + if (ptr->d_type != DT_REG) { + continue; + } + if (!hex_string_to_int(ptr->d_name, &segnum)) { + closedir(dir); + return; + } + /* one segment file has 8k*8bit/2*32 xids */ + uint32 segnum_xid = BLCKSZ * CLOG_XACTS_PER_BYTE * SLRU_PAGES_PER_SEGMENT; + /* the first xid number of current segment file */ + TransactionId xid = (uint64)segnum * segnum_xid; + pg_atomic_write_u64(&g_instance.undo_cxt.hotStandbyRecycleXid, xid); + ereport(LOG, (errmsg("Startup: write hotStandbyRecycleXid %lu", xid))); + + closedir(dir); + return; + } + closedir(dir); +} + /* * This must be called ONCE during postmaster or standalone-backend startup */ @@ -9520,6 +9584,7 @@ void StartupXLOG(void) } else { pg_atomic_write_u64(&g_instance.undo_cxt.globalRecycleXid, InvalidTransactionId); } + set_hot_standby_recycle_xid(); /* * Initialize replication slots, before there's a chance to remove @@ -9732,6 +9797,9 @@ void StartupXLOG(void) "have to use another backup for recovery."))); } t_thrd.shemem_ptr_cxt.ControlFile->backupEndPoint = t_thrd.shemem_ptr_cxt.ControlFile->minRecoveryPoint; + ereport(LOG, (errmsg("backup_from_standby: set backup end point to %X/%X", + (uint32)(t_thrd.shemem_ptr_cxt.ControlFile->backupEndPoint >> 32), + (uint32)t_thrd.shemem_ptr_cxt.ControlFile->backupEndPoint))); } else if (backupFromRoach) { t_thrd.shemem_ptr_cxt.ControlFile->backupEndPoint = t_thrd.shemem_ptr_cxt.ControlFile->minRecoveryPoint; ereport(LOG, (errmsg("perform roach backup restore and set backup end point to %X/%X", @@ -9911,13 +9979,14 @@ void StartupXLOG(void) * redo LSN and future consistent point. */ ereport(LOG, (errmsg("redo minRecoveryPoint at %X/%X; backupStartPoint at %X/%X; " - "backupEndRequired %s", + "backupEndPoint at %X/%X;backupEndRequired %s", (uint32)(t_thrd.xlog_cxt.minRecoveryPoint >> 32), (uint32)t_thrd.xlog_cxt.minRecoveryPoint, (uint32)(t_thrd.shemem_ptr_cxt.ControlFile->backupStartPoint >> 32), (uint32)t_thrd.shemem_ptr_cxt.ControlFile->backupStartPoint, + (uint32)(t_thrd.shemem_ptr_cxt.ControlFile->backupEndPoint >> 32), + (uint32)t_thrd.shemem_ptr_cxt.ControlFile->backupEndPoint, t_thrd.shemem_ptr_cxt.ControlFile->backupEndRequired ? "TRUE" : "FALSE"))); - init_extreme_rto_standby_read_first_snapshot(checkPoint.redo); pg_atomic_write_u32(&t_thrd.walreceiverfuncs_cxt.WalRcv->rcvDoneFromShareStorage, false); // Allow read-only connections immediately if we're consistent already. @@ -10904,7 +10973,7 @@ void ArchiveXlogForForceFinishRedo(XLogReaderState *xlogreader, TermFileData *te void backup_cut_xlog_file(XLogRecPtr lastReplayedEndRecPtr) { errno_t errorno = EOK; - ereport(DEBUG1, (errmsg("end of backup reached"))); + ereport(LOG, (errmsg("end of backup reached"))); LWLockAcquire(ControlFileLock, LW_EXCLUSIVE); @@ -13210,7 +13279,8 @@ static void KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo, XLogRecPtr curIns } } - if (RecoveryInProgress() && IS_EXRTO_READ) { + if (RecoveryInProgress() && IS_EXRTO_READ && !dummyStandbyMode && + !g_instance.attr.attr_storage.enable_exrto_standby_read_opt) { XLogRecPtr recycle_recptr = pg_atomic_read_u64(&g_instance.comm_cxt.predo_cxt.global_recycle_lsn); XLogSegNo recyle_segno; XLByteToSeg(recycle_recptr, recyle_segno); @@ -13723,7 +13793,8 @@ void xlog_redo(XLogReaderState *record) rc = memcpy_s(&startpoint, sizeof(startpoint), XLogRecGetData(record), sizeof(startpoint)); securec_check(rc, "", ""); - if (XLByteEQ(t_thrd.shemem_ptr_cxt.ControlFile->backupStartPoint, startpoint)) { + if (XLByteEQ(t_thrd.shemem_ptr_cxt.ControlFile->backupStartPoint, startpoint) && + t_thrd.shemem_ptr_cxt.ControlFile->backupEndRequired) { /* * We have reached the end of base backup, the point where * pg_stop_backup() was done. The data on disk is now consistent. @@ -13731,7 +13802,7 @@ void xlog_redo(XLogReaderState *record) * sure we don't allow starting up at an earlier point even if * recovery is stopped and restarted soon after this. */ - ereport(DEBUG1, (errmsg("end of backup reached"))); + ereport(LOG, (errmsg("end of backup reached"))); LWLockAcquire(ControlFileLock, LW_EXCLUSIVE); diff --git a/src/gausskernel/storage/access/ubtree/ubtpage.cpp b/src/gausskernel/storage/access/ubtree/ubtpage.cpp index f28c2b850..55b6a8a38 100644 --- a/src/gausskernel/storage/access/ubtree/ubtpage.cpp +++ b/src/gausskernel/storage/access/ubtree/ubtpage.cpp @@ -25,6 +25,7 @@ #include "knl/knl_variable.h" #include "access/hio.h" +#include "access/multi_redo_api.h" #include "access/nbtree.h" #include "access/ubtree.h" #include "access/transam.h" @@ -165,7 +166,8 @@ Buffer UBTreeGetRoot(Relation rel, int access) valid = PinBuffer(buf, NULL); if (valid) { LockBuffer(rootbuf, BT_READ); - isRootCacheValid = RelFileNodeEquals(buf->tag.rnode, rel->rd_node) && (buf->tag.blockNum == rootblkno); + isRootCacheValid = (!IS_EXRTO_STANDBY_READ) && RelFileNodeEquals(buf->tag.rnode, rel->rd_node) && + (buf->tag.blockNum == rootblkno); if (!isRootCacheValid) UnlockReleaseBuffer(rootbuf); } else { diff --git a/src/gausskernel/storage/access/ubtree/ubtsearch.cpp b/src/gausskernel/storage/access/ubtree/ubtsearch.cpp index 78d901461..22893d32f 100644 --- a/src/gausskernel/storage/access/ubtree/ubtsearch.cpp +++ b/src/gausskernel/storage/access/ubtree/ubtsearch.cpp @@ -148,7 +148,7 @@ BTStack UBTreeSearch(Relation rel, BTScanInsert key, Buffer *bufP, int access, b ExecuteUstoreVerify(USTORE_VERIFY_MOD_UBTREE, (char *) &verifyParams); } /* drop the read lock on the parent page, acquire one on the child */ - *bufP = _bt_relandgetbuf(rel, *bufP, blkno, pageAccess); + *bufP = _bt_relandgetbuf(rel, *bufP, blkno, pageAccess, par_blkno); /* okay, all set to move down a level */ stack_in = new_stack; diff --git a/src/gausskernel/storage/access/ubtree/ubtutils.cpp b/src/gausskernel/storage/access/ubtree/ubtutils.cpp index 8fbf0c786..505401eb4 100644 --- a/src/gausskernel/storage/access/ubtree/ubtutils.cpp +++ b/src/gausskernel/storage/access/ubtree/ubtutils.cpp @@ -481,6 +481,10 @@ bool UBTreeItupGetXminXmax(Page page, OffsetNumber offnum, TransactionId oldest_ if (!TransactionIdIsValid(oldest_xmin)) { oldest_xmin = u_sess->utils_cxt.RecentGlobalDataXmin; } + /* we can't do bypass in hotstandby read mode, or there will be different between index scan and seq scan */ + if (RecoveryInProgress()) { + oldest_xmin = InvalidTransactionId; + } if (!TransactionIdIsValid(*xmin)) { isDead = true; diff --git a/src/gausskernel/storage/access/ustore/knl_uextremeredo.cpp b/src/gausskernel/storage/access/ustore/knl_uextremeredo.cpp index 5797e42a4..e402ed226 100644 --- a/src/gausskernel/storage/access/ustore/knl_uextremeredo.cpp +++ b/src/gausskernel/storage/access/ustore/knl_uextremeredo.cpp @@ -201,7 +201,7 @@ static XLogRecParseState *UHeapXlogFreezeTDParseBlock(XLogReaderState *record, u if (recordstatehead == NULL) { return NULL; } - XLogRecSetBlockDataState(record, UHEAP_FREEZE_TD_ORIG_BLOCK_NUM, recordstatehead); + XLogRecSetBlockDataState(record, UHEAP_FREEZE_TD_ORIG_BLOCK_NUM, recordstatehead, BLOCK_DATA_MAIN_DATA_TYPE, true); return recordstatehead; } @@ -213,7 +213,7 @@ static XLogRecParseState *UHeapXlogInvalidTDParseBlock(XLogReaderState *record, if (recordstatehead == NULL) { return NULL; } - XLogRecSetBlockDataState(record, UHEAP_INVALID_TD_ORIG_BLOCK_NUM, recordstatehead); + XLogRecSetBlockDataState(record, UHEAP_INVALID_TD_ORIG_BLOCK_NUM, recordstatehead, BLOCK_DATA_MAIN_DATA_TYPE, true); return recordstatehead; } @@ -225,7 +225,7 @@ static XLogRecParseState *UHeapXlogCleanParseBlock(XLogReaderState *record, uint if (recordstatehead == NULL) { return NULL; } - XLogRecSetBlockDataState(record, UHEAP_CLEAN_ORIG_BLOCK_NUM, recordstatehead); + XLogRecSetBlockDataState(record, UHEAP_CLEAN_ORIG_BLOCK_NUM, recordstatehead, BLOCK_DATA_MAIN_DATA_TYPE, true); return recordstatehead; } @@ -276,7 +276,7 @@ static XLogRecParseState *UHeap2XlogBaseShiftParseBlock(XLogReaderState *record, if (recordstatehead == NULL) { return NULL; } - XLogRecSetBlockDataState(record, UHEAP2_ORIG_BLOCK_NUM, recordstatehead); + XLogRecSetBlockDataState(record, UHEAP2_ORIG_BLOCK_NUM, recordstatehead, BLOCK_DATA_MAIN_DATA_TYPE, true); return recordstatehead; } @@ -288,7 +288,7 @@ static XLogRecParseState *UHeap2XlogFreezeParseBlock(XLogReaderState *record, ui if (recordstatehead == NULL) { return NULL; } - XLogRecSetBlockDataState(record, UHEAP2_ORIG_BLOCK_NUM, recordstatehead); + XLogRecSetBlockDataState(record, UHEAP2_ORIG_BLOCK_NUM, recordstatehead, BLOCK_DATA_MAIN_DATA_TYPE, true); return recordstatehead; } @@ -404,7 +404,7 @@ static XLogRecParseState *UHeapXlogUheapUndoResetSlotParseBlock(XLogReaderState if (recordstatehead == NULL) { return NULL; } - XLogRecSetBlockDataState(record, UHEAP_UNDOACTION_ORIG_BLOCK_NUM, recordstatehead); + XLogRecSetBlockDataState(record, UHEAP_UNDOACTION_ORIG_BLOCK_NUM, recordstatehead, BLOCK_DATA_MAIN_DATA_TYPE, true); return recordstatehead; } @@ -416,7 +416,7 @@ static XLogRecParseState *UHeapXlogUheapUndoPageParseBlock(XLogReaderState *reco if (recordstatehead == NULL) { return NULL; } - XLogRecSetBlockDataState(record, UHEAP_UNDOACTION_ORIG_BLOCK_NUM, recordstatehead); + XLogRecSetBlockDataState(record, UHEAP_UNDOACTION_ORIG_BLOCK_NUM, recordstatehead, BLOCK_DATA_MAIN_DATA_TYPE, true); return recordstatehead; } @@ -428,7 +428,7 @@ static XLogRecParseState *UHeapXlogUheapUndoAbortSpecInsertParseBlock(XLogReader if (recordstatehead == NULL) { return NULL; } - XLogRecSetBlockDataState(record, UHEAP_UNDOACTION_ORIG_BLOCK_NUM, recordstatehead); + XLogRecSetBlockDataState(record, UHEAP_UNDOACTION_ORIG_BLOCK_NUM, recordstatehead, BLOCK_DATA_MAIN_DATA_TYPE, true); return recordstatehead; } diff --git a/src/gausskernel/storage/access/ustore/knl_uundorecord.cpp b/src/gausskernel/storage/access/ustore/knl_uundorecord.cpp index ad1ad7dde..db419459c 100644 --- a/src/gausskernel/storage/access/ustore/knl_uundorecord.cpp +++ b/src/gausskernel/storage/access/ustore/knl_uundorecord.cpp @@ -546,7 +546,8 @@ UndoTraversalState FetchUndoRecord(__inout UndoRecord *urec, _in_ SatisfyUndoRec return UNDO_TRAVERSAL_ABORT; } - if (isNeedBypass && TransactionIdPrecedes(urec->Xid(), g_instance.undo_cxt.globalFrozenXid)) { + if (isNeedBypass && TransactionIdPrecedes(urec->Xid(), g_instance.undo_cxt.globalFrozenXid) && + !RecoveryInProgress()) { ereport(DEBUG1, (errmsg(UNDOFORMAT("Check visibility by globalFrozenXid")))); return UNDO_TRAVERSAL_STOP; } diff --git a/src/gausskernel/storage/access/ustore/knl_uvisibility.cpp b/src/gausskernel/storage/access/ustore/knl_uvisibility.cpp index a12abb556..0f01a3834 100644 --- a/src/gausskernel/storage/access/ustore/knl_uvisibility.cpp +++ b/src/gausskernel/storage/access/ustore/knl_uvisibility.cpp @@ -249,9 +249,9 @@ bool UHeapTupleSatisfiesVisibility(UHeapTuple uhtup, Snapshot snapshot, Buffer b } uint64 globalFrozenXid = isFlashBack ? pg_atomic_read_u64(&g_instance.undo_cxt.globalRecycleXid) : pg_atomic_read_u64(&g_instance.undo_cxt.globalFrozenXid); - if (pm_state_is_hot_standby()) { + if (RecoveryInProgress()) { /* in hot standby mode, if globalRecycleXid advance during query, it may cause data inconsistency */ - globalFrozenXid = 0; + globalFrozenXid = pg_atomic_read_u64(&g_instance.undo_cxt.hotStandbyRecycleXid); } if (TransactionIdIsValid(tdinfo.xid) && TransactionIdPrecedes(tdinfo.xid, globalFrozenXid)) { /* The slot is old enough that we can treat it as frozen. */ @@ -817,9 +817,9 @@ bool UHeapTupleFetch(Relation rel, Buffer buffer, OffsetNumber offnum, Snapshot uint64 oldestRecycleXidHavingUndo = pg_atomic_read_u64(&g_instance.undo_cxt.globalRecycleXid); uint64 oldestXidHavingUndo = (isFlashBack || isLogical) ? oldestRecycleXidHavingUndo : pg_atomic_read_u64(&g_instance.undo_cxt.globalFrozenXid); - if (pm_state_is_hot_standby()) { + if (RecoveryInProgress()) { /* in hot standby mode, if globalRecycleXid advance during query, it may cause data inconsistency */ - oldestXidHavingUndo = 0; + oldestXidHavingUndo = pg_atomic_read_u64(&g_instance.undo_cxt.hotStandbyRecycleXid); } if (TransactionIdIsValid(tdinfo.xid) && TransactionIdPrecedes(tdinfo.xid, oldestXidHavingUndo)) { if (TransactionIdOlderThanAllUndo(tdinfo.xid)) { diff --git a/src/gausskernel/storage/access/ustore/undo/knl_uundorecycle.cpp b/src/gausskernel/storage/access/ustore/undo/knl_uundorecycle.cpp index b6201db3e..e2d5e33d9 100755 --- a/src/gausskernel/storage/access/ustore/undo/knl_uundorecycle.cpp +++ b/src/gausskernel/storage/access/ustore/undo/knl_uundorecycle.cpp @@ -44,6 +44,7 @@ #include "utils/postinit.h" #include "utils/gs_bitmap.h" #include "pgstat.h" +#include "access/ustore/knl_uvisibility.h" #define TRANS_PARTITION_LINEAR_SPARE_TIME(degree) \ (degree > 3000 ? 3000 : degree) @@ -561,7 +562,20 @@ void exrto_standby_release_space(UndoZone *zone, TransactionId recycle_xid, Undo zone->ReleaseSpace(start_undo_ptr, end_undo_ptr, &g_forceRecycleSize); zone->ReleaseSlotSpace(0, recycle_exrto, &g_forceRecycleSize); } - + +bool is_undo_slot_exist(UndoSlotPtr slot_ptr) +{ + bool ret = false; + RelFileNode rnode; + UNDO_PTR_ASSIGN_REL_FILE_NODE(rnode, slot_ptr, UNDO_SLOT_DB_OID); + SMgrRelation reln = smgropen(rnode, InvalidBackendId); + if (smgrexists(reln, UNDO_FORKNUM, (BlockNumber)UNDO_PTR_GET_BLOCK_NUM(slot_ptr))) { + ret = true; + } + smgrclose(reln); + return ret; +} + bool exrto_standby_recycle_space(UndoZone *zone, TransactionId recycle_xmin) { UndoSlotPtr recycle_exrto = zone->get_recycle_tslot_ptr_exrto(); @@ -578,6 +592,23 @@ bool exrto_standby_recycle_space(UndoZone *zone, TransactionId recycle_xmin) zone->GetZoneId(), recycle_xmin, recycle_exrto, recycle_primary))); while (recycle_exrto < recycle_primary) { + uint64 start_segno = (uint)((UNDO_PTR_GET_OFFSET(recycle_exrto)) / UNDO_META_SEGMENT_SIZE); + uint64 end_segno = (uint)((UNDO_PTR_GET_OFFSET(recycle_exrto)) / UNDO_META_SEGMENT_SIZE + 1); + if (!is_undo_slot_exist(recycle_exrto)) { + zone->ForgetUndoBuffer( + start_segno * UNDO_META_SEGMENT_SIZE, end_segno * UNDO_META_SEGMENT_SIZE, UNDO_DB_OID); + ereport(WARNING, + (errmodule(MOD_UNDO), + errmsg(UNDOFORMAT("exrto_standby_recycle_space zone_id:%d, recycle_xmin:%lu, recycle_exrto:%lu, " + "recycle_primary:%lu, undo slot not exist."), + zone->GetZoneId(), + recycle_xmin, + recycle_exrto, + recycle_primary))); + recycle_exrto = GetNextSlotPtr(recycle_exrto); + continue; + } + UndoSlotBuffer& slot_buf = g_slotBufferCache->FetchTransactionBuffer(recycle_exrto); UndoRecPtr start_undo_ptr = INVALID_UNDO_REC_PTR; start = recycle_exrto; @@ -631,7 +662,7 @@ bool exrto_standby_recycle_undo_zone() if (g_instance.undo_cxt.uZoneCount == 0 || g_instance.undo_cxt.uZones == NULL) { return recycled; } - TransactionId recycle_xmin = extreme_rto::exrto_calculate_recycle_xmin_for_undo(); + TransactionId recycle_xmin = exrto_calculate_recycle_xmin_for_undo(); for (idx = 0; idx < PERSIST_ZONE_COUNT && !t_thrd.undorecycler_cxt.shutdown_requested; idx++) { UndoZone *zone = (UndoZone *)g_instance.undo_cxt.uZones[idx]; if (zone == NULL) { @@ -809,6 +840,7 @@ void UndoRecycleMain() ereport(LOG, (errmodule(MOD_UNDO), errmsg(UNDOFORMAT("sleep 10s, ensure the snapcapturer can give the undorecyclemain a valid recycleXmin.")))); exrto_recycle_residual_undo_file("recycle_main"); + t_thrd.undorecycler_cxt.is_recovery_in_progress = RecoveryInProgress(); while (true) { if (t_thrd.undorecycler_cxt.got_SIGHUP) { t_thrd.undorecycler_cxt.got_SIGHUP = false; @@ -817,7 +849,14 @@ void UndoRecycleMain() if (t_thrd.undorecycler_cxt.shutdown_requested) { ShutDownRecycle(recycleMaxXIDs); } - if (!RecoveryInProgress()) { + bool is_in_progress = RecoveryInProgress(); + if (is_in_progress != t_thrd.undorecycler_cxt.is_recovery_in_progress) { + ereport(LOG, (errmodule(MOD_UNDO), + errmsg(UNDOFORMAT("recycle_main: stop undo recycler because recovery_in_progress change " + "from %u to %u."), t_thrd.undorecycler_cxt.is_recovery_in_progress, is_in_progress))); + ShutDownRecycle(recycleMaxXIDs); + } + if (!t_thrd.undorecycler_cxt.is_recovery_in_progress) { TransactionId recycleXmin = InvalidTransactionId; TransactionId oldestXmin = GetOldestXminForUndo(&recycleXmin); if (!TransactionIdIsValid(recycleXmin) || diff --git a/src/gausskernel/storage/access/ustore/undo/knl_uundospace.cpp b/src/gausskernel/storage/access/ustore/undo/knl_uundospace.cpp index 3d60a8a4d..49f796f75 100644 --- a/src/gausskernel/storage/access/ustore/undo/knl_uundospace.cpp +++ b/src/gausskernel/storage/access/ustore/undo/knl_uundospace.cpp @@ -114,7 +114,7 @@ void UndoSpace::UnlinkUndoLog(int zid, UndoLogOffset offset, uint32 dbId) RelFileNode rnode; UndoLogOffset head; UndoLogOffset old_head; - if (IS_EXRTO_STANDBY_READ) { + if (t_thrd.undorecycler_cxt.is_recovery_in_progress) { head = head_exrto; old_head = head_exrto; set_head_exrto(offset); diff --git a/src/gausskernel/storage/access/ustore/undo/knl_uundozone.cpp b/src/gausskernel/storage/access/ustore/undo/knl_uundozone.cpp index 6ae2a8700..2408ff1a1 100644 --- a/src/gausskernel/storage/access/ustore/undo/knl_uundozone.cpp +++ b/src/gausskernel/storage/access/ustore/undo/knl_uundozone.cpp @@ -280,7 +280,7 @@ void UndoZone::ReleaseSpace(UndoRecPtr starturp, UndoRecPtr endurp, int *forceRe UndoLogOffset end = UNDO_PTR_GET_OFFSET(endurp); int startSegno; UndoLogOffset head; - if (IS_EXRTO_STANDBY_READ) { + if (t_thrd.undorecycler_cxt.is_recovery_in_progress) { head = undoSpace_.Head_exrto(); } else { head = undoSpace_.Head(); @@ -297,7 +297,7 @@ void UndoZone::ReleaseSpace(UndoRecPtr starturp, UndoRecPtr endurp, int *forceRe UndoRecPtr prevHead = MAKE_UNDO_PTR(zid_, head); undoSpace_.UnlinkUndoLog(zid_, endSegno * UNDO_LOG_SEGMENT_SIZE, UNDO_DB_OID); Assert(undoSpace_.Head() <= insertURecPtr_); - if (pLevel_ == UNDO_PERMANENT && (!IS_EXRTO_STANDBY_READ)) { + if (pLevel_ == UNDO_PERMANENT && (!t_thrd.undorecycler_cxt.is_recovery_in_progress)) { START_CRIT_SECTION(); undoSpace_.MarkDirty(); XlogUndoUnlink undoUnlink; @@ -340,7 +340,7 @@ void UndoZone::ReleaseSlotSpace(UndoRecPtr startSlotPtr, UndoRecPtr endSlotPtr, { UndoLogOffset end = UNDO_PTR_GET_OFFSET(endSlotPtr); UndoLogOffset head; - if (IS_EXRTO_STANDBY_READ) { + if (t_thrd.undorecycler_cxt.is_recovery_in_progress) { head = slotSpace_.Head_exrto(); } else { head = slotSpace_.Head(); @@ -357,7 +357,7 @@ void UndoZone::ReleaseSlotSpace(UndoRecPtr startSlotPtr, UndoRecPtr endSlotPtr, UndoRecPtr prevHead = MAKE_UNDO_PTR(zid_, head); slotSpace_.UnlinkUndoLog(zid_, endSegno * UNDO_META_SEGMENT_SIZE, UNDO_SLOT_DB_OID); Assert(slotSpace_.Head() <= allocateTSlotPtr_); - if (pLevel_ == UNDO_PERMANENT && !(IS_EXRTO_STANDBY_READ)) { + if (pLevel_ == UNDO_PERMANENT && !(t_thrd.undorecycler_cxt.is_recovery_in_progress)) { START_CRIT_SECTION(); slotSpace_.MarkDirty(); XlogUndoUnlink undoUnlink; diff --git a/src/gausskernel/storage/buffer/bufmgr.cpp b/src/gausskernel/storage/buffer/bufmgr.cpp index 65601d55a..27375bc52 100644 --- a/src/gausskernel/storage/buffer/bufmgr.cpp +++ b/src/gausskernel/storage/buffer/bufmgr.cpp @@ -6197,13 +6197,11 @@ bool HoldingBufferPinThatDelaysRecovery(void) if (IS_EXRTO_READ) { return false; } - SpinLockAcquire(&(g_instance.comm_cxt.predo_cxt.destroy_lock)); int bufids[MAX_RECOVERY_THREAD_NUM + 1]; errno_t rc = memset_s(bufids, sizeof(bufids), -1, sizeof(bufids)); securec_check(rc, "\0", "\0"); uint32 bufLen = parallel_recovery::GetStartupBufferPinWaitBufLen(); parallel_recovery::GetStartupBufferPinWaitBufId(bufids, bufLen); - SpinLockRelease(&(g_instance.comm_cxt.predo_cxt.destroy_lock)); for (uint32 i = 0; i < bufLen; i++) { /* diff --git a/src/gausskernel/storage/ipc/procarray.cpp b/src/gausskernel/storage/ipc/procarray.cpp index ddcbd8f70..5095d235f 100755 --- a/src/gausskernel/storage/ipc/procarray.cpp +++ b/src/gausskernel/storage/ipc/procarray.cpp @@ -1341,8 +1341,12 @@ bool TransactionIdIsInProgress(TransactionId xid, uint32* needSync, bool shortcu assigned value * local must sync with gtm. */ - if (shortcutByRecentXmin && - TransactionIdPrecedes(xid, pg_atomic_read_u64(&g_instance.undo_cxt.globalRecycleXid))) { + uint64 recycle_xid = pg_atomic_read_u64(&g_instance.undo_cxt.globalRecycleXid); + /* in hotstandby mode, the proc may being runnnig */ + if (RecoveryInProgress()) { + recycle_xid = InvalidTransactionId; + } + if (shortcutByRecentXmin && TransactionIdPrecedes(xid, recycle_xid)) { xc_by_recent_xmin_inc(); /* @@ -2114,6 +2118,8 @@ Snapshot GetSnapshotData(Snapshot snapshot, bool force_local_snapshot) bool retry_get = false; uint64 retry_count = 0; const static uint64 WAIT_COUNT = 0x7FFFF; + /* reset xmin before acquiring lwlock, in case blocking redo */ + t_thrd.pgxact->xmin = InvalidTransactionId; RETRY_GET: if (snapshot->takenDuringRecovery && !StreamThreadAmI() && !IS_EXRTO_READ && !u_sess->proc_cxt.clientIsCMAgent) { @@ -2433,15 +2439,15 @@ Snapshot GetSnapshotData(Snapshot snapshot, bool force_local_snapshot) snapshot->copied = false; if (snapshot->takenDuringRecovery) { - (void)pgstat_report_waitstatus(oldStatus); if (IsDefaultExtremeRtoMode() && IS_EXRTO_STANDBY_READ) { - extreme_rto::exrto_read_snapshot(snapshot); + exrto_read_snapshot(snapshot); if (t_thrd.proc->exrto_reload_cache) { t_thrd.proc->exrto_reload_cache = false; reset_invalidation_cache(); } AcceptInvalidationMessages(); } + (void)pgstat_report_waitstatus(oldStatus); } return snapshot; @@ -2724,6 +2730,9 @@ TransactionId GetOldestActiveTransactionId(TransactionId *globalXmin) xmin = oldestRunningXid; } *globalXmin = xmin; + if (IS_EXRTO_STANDBY_READ) { + ereport(LOG, (errmsg("proc_array_get_oldest_active_transaction_id: global_xmin = %lu", *globalXmin))); + } return oldestRunningXid; } @@ -3236,7 +3245,8 @@ ThreadId CancelVirtualTransaction(const VirtualTransactionId& vxid, ProcSignalRe return pid; } -bool proc_array_cancel_conflicting_proc(TransactionId latest_removed_xid, bool reach_max_check_times) +bool proc_array_cancel_conflicting_proc( + TransactionId latest_removed_xid, XLogRecPtr truncate_redo_lsn, bool reach_max_check_times) { ProcArrayStruct* proc_array = g_instance.proc_array_idx; bool conflict = false; @@ -3262,8 +3272,9 @@ bool proc_array_cancel_conflicting_proc(TransactionId latest_removed_xid, bool r continue; } - /* cancel query when its xmin < latest_removed_xid */ - if (TransactionIdPrecedesOrEquals(pxmin, latest_removed_xid)) { + /* cancel query when its xmin < latest_removed_xid */ + if (TransactionIdPrecedesOrEquals(pxmin, latest_removed_xid) || + (truncate_redo_lsn != InvalidXLogRecPtr && XLByteLT(read_lsn, truncate_redo_lsn))) { conflict = true; pg_proc->recoveryConflictPending = true; if (pg_proc->pid != 0) { @@ -3277,9 +3288,16 @@ bool proc_array_cancel_conflicting_proc(TransactionId latest_removed_xid, bool r * an unresponsive backend when system is heavily loaded. */ ereport(LOG, - (errmsg(EXRTOFORMAT("exrto_gen_snap_time: %ld, current_timestamp: %ld, cancel thread while " - "redo truncate, thread id = %lu\n"), - pg_proc->exrto_gen_snap_time, GetCurrentTimestamp(), pg_proc->pid))); + (errmsg(EXRTOFORMAT("cancel thread while " + "redo truncate (lsn: %08X/%08X, latest_removed_xid: %lu), thread id = %lu, " + "read_lsn: %08X/%08X, xmin: %lu"), + (uint32)(truncate_redo_lsn >> UINT64_HALF), + (uint32)truncate_redo_lsn, + latest_removed_xid, + pg_proc->pid, + (uint32)(read_lsn >> UINT64_HALF), + (uint32)read_lsn, + pxmin))); pg_usleep(5000L); } } diff --git a/src/gausskernel/storage/ipc/sinvaladt.cpp b/src/gausskernel/storage/ipc/sinvaladt.cpp index 490f582c4..f70fbfcec 100644 --- a/src/gausskernel/storage/ipc/sinvaladt.cpp +++ b/src/gausskernel/storage/ipc/sinvaladt.cpp @@ -20,6 +20,7 @@ #include #include "miscadmin.h" +#include "access/multi_redo_api.h" #include "storage/backendid.h" #include "storage/ipc.h" #include "storage/proc.h" @@ -739,16 +740,18 @@ int SIGetDataEntries(SharedInvalidationMessage* data, int datasize, bool workses n = 0; XLogRecPtr read_lsn = InvalidXLogRecPtr; - if (u_sess->utils_cxt.CurrentSnapshot != NULL && - XLogRecPtrIsValid(u_sess->utils_cxt.CurrentSnapshot->read_lsn)) { - read_lsn = u_sess->utils_cxt.CurrentSnapshot->read_lsn; - } else if (XLogRecPtrIsValid(t_thrd.proc->exrto_read_lsn)) { - read_lsn = t_thrd.proc->exrto_read_lsn; + if (IS_EXRTO_STANDBY_READ) { + if (u_sess->utils_cxt.CurrentSnapshot != NULL && + XLogRecPtrIsValid(u_sess->utils_cxt.CurrentSnapshot->read_lsn)) { + read_lsn = u_sess->utils_cxt.CurrentSnapshot->read_lsn; + } else if (XLogRecPtrIsValid(t_thrd.proc->exrto_read_lsn)) { + read_lsn = t_thrd.proc->exrto_read_lsn; + } } while (n < datasize && stateP->nextMsgNum < max) { int index = stateP->nextMsgNum % MAXNUMMESSAGES; - if (read_lsn != InvalidXLogRecPtr && segP->buffer[index].lsn != InvalidXLogRecPtr) { + if (XLogRecPtrIsValid(read_lsn) && XLogRecPtrIsValid(segP->buffer[index].lsn)) { if (XLByteLT(read_lsn, segP->buffer[index].lsn)) { break; } diff --git a/src/gausskernel/storage/ipc/standby.cpp b/src/gausskernel/storage/ipc/standby.cpp index d02342920..cbab224f7 100755 --- a/src/gausskernel/storage/ipc/standby.cpp +++ b/src/gausskernel/storage/ipc/standby.cpp @@ -35,13 +35,14 @@ #include "utils/timestamp.h" #include "utils/snapmgr.h" #include "pgxc/poolutils.h" +#include "catalog/pg_partition_fn.h" #include "replication/walreceiver.h" static void ResolveRecoveryConflictWithVirtualXIDs(VirtualTransactionId* waitlist, TransactionId* xminArray, ProcSignalReason reason, TimestampTz waitStart, TransactionId limitXmin = InvalidTransactionId); static void ResolveRecoveryConflictWithLock(Oid dbOid, Oid relOid); static void LogAccessExclusiveLocks(int nlocks, xl_standby_lock* locks); -static void LogReleaseAccessExclusiveLocks(int nlocks, xl_standby_lock* locks); +static void log_access_exclusive_locks_new(int nlocks, XlStandbyLockNew* locks); static XLogRecPtr LogCurrentRunningXacts(RunningTransactions CurrRunningXacts); static void RecordCommittingCsnInfo(TransactionId xid); @@ -198,6 +199,9 @@ static void ResolveRecoveryConflictWithVirtualXIDs(VirtualTransactionId* waitlis /* wait until the virtual xid is gone */ while (!VirtualXactLock(*waitlist, false)) { PGPROC* proc = BackendIdGetProc((*waitlist).backendId); + if (proc == NULL) { + break; + } PGXACT* pgxact = &g_instance.proc_base_all_xacts[proc->pgprocno]; if (xminArray != NULL && pgxact->xmin != *xminArray && (!TransactionIdIsValid(pgxact->xmin) || TransactionIdFollows(pgxact->xmin, limitXmin))) { @@ -608,7 +612,7 @@ void CheckRecoveryConflictDeadlock(void) * We use session locks rather than normal locks so we don't need * ResourceOwners. */ -void StandbyAcquireAccessExclusiveLock(TransactionId xid, Oid dbOid, Oid relOid) +void StandbyAcquireAccessExclusiveLock(TransactionId xid, Oid dbOid, Oid relOid, uint32 seq) { LOCKTAG locktag; @@ -651,13 +655,22 @@ void StandbyAcquireAccessExclusiveLock(TransactionId xid, Oid dbOid, Oid relOid) entry->locks = NIL; } - xl_standby_lock* newlock = (xl_standby_lock*)palloc(sizeof(xl_standby_lock)); + XlStandbyLockNew* newlock = (XlStandbyLockNew*)palloc(sizeof(XlStandbyLockNew)); newlock->xid = xid; newlock->dbOid = dbOid; newlock->relOid = relOid; + newlock->seq = seq; entry->locks = lappend(entry->locks, newlock); - SET_LOCKTAG_RELATION(locktag, newlock->dbOid, newlock->relOid); + if (seq != InvalidOid) { + if (seq == PARTITION_OBJECT_LOCK_SDEQUENCE || seq == INTERVAL_PARTITION_LOCK_SDEQUENCE) { + SET_LOCKTAG_OBJECT(locktag, newlock->dbOid, newlock->relOid, newlock->seq, 0); + } else { + SET_LOCKTAG_PARTITION(locktag, newlock->dbOid, newlock->relOid, newlock->seq); + } + } else { + SET_LOCKTAG_RELATION(locktag, newlock->dbOid, newlock->relOid); + } if (LockAcquireExtended(&locktag, AccessExclusiveLock, true, true, false) == LOCKACQUIRE_NOT_AVAIL) ResolveRecoveryConflictWithLock(newlock->dbOid, newlock->relOid); @@ -666,11 +679,20 @@ void StandbyAcquireAccessExclusiveLock(TransactionId xid, Oid dbOid, Oid relOid) static void StandbyReleaseLockList(List *locks) { while (locks) { - xl_standby_lock *lock = (xl_standby_lock *) linitial(locks); + XlStandbyLockNew *lock = (XlStandbyLockNew *) linitial(locks); LOCKTAG locktag; - ereport(trace_recovery(DEBUG4), (errmsg("releasing recovery lock: xid %lu db %u rel %u", lock->xid, - lock->dbOid, lock->relOid))); - SET_LOCKTAG_RELATION(locktag, lock->dbOid, lock->relOid); + ereport(trace_recovery(DEBUG4), (errmsg("releasing recovery lock: xid %lu db %u rel %u seq %u", lock->xid, + lock->dbOid, lock->relOid, lock->seq))); + if (lock->seq != InvalidOid) { + if (lock->seq == PARTITION_OBJECT_LOCK_SDEQUENCE || lock->seq == INTERVAL_PARTITION_LOCK_SDEQUENCE) { + SET_LOCKTAG_OBJECT(locktag, lock->dbOid, lock->relOid, lock->seq, 0); + } else { + SET_LOCKTAG_PARTITION(locktag, lock->dbOid, lock->relOid, lock->seq); + } + } else { + SET_LOCKTAG_RELATION(locktag, lock->dbOid, lock->relOid); + } + if (!LockRelease(&locktag, AccessExclusiveLock, true)) { ereport(LOG, (errmsg("RecoveryLockLists contains entry for lock no longer recorded by lock manager: " "xid %lu database %u relation %u", lock->xid, lock->dbOid, lock->relOid))); @@ -833,11 +855,19 @@ void standby_redo(XLogReaderState* record) return; if (info == XLOG_STANDBY_LOCK) { - xl_standby_locks* xlrec = (xl_standby_locks*)XLogRecGetData(record); - int i; - - for (i = 0; i < xlrec->nlocks; i++) - StandbyAcquireAccessExclusiveLock(xlrec->locks[i].xid, xlrec->locks[i].dbOid, xlrec->locks[i].relOid); + if ((XLogRecGetInfo(record) & PARTITION_ACCESS_EXCLUSIVE_LOCK_UPGRADE_FLAG) == 0) { + xl_standby_locks *xlrec = (xl_standby_locks *)XLogRecGetData(record); + for (int i = 0; i < xlrec->nlocks; i++) { + StandbyAcquireAccessExclusiveLock(xlrec->locks[i].xid, xlrec->locks[i].dbOid, xlrec->locks[i].relOid, + InvalidOid); + } + } else { + XLogStandbyLocksNew *xlrec = (XLogStandbyLocksNew *)XLogRecGetData(record); + for (int i = 0; i < xlrec->nlocks; i++) { + StandbyAcquireAccessExclusiveLock(xlrec->locks[i].xid, xlrec->locks[i].dbOid, xlrec->locks[i].relOid, + xlrec->locks[i].seq); + } + } } else if (info == XLOG_RUNNING_XACTS) { RunningTransactionsData running; @@ -1063,28 +1093,50 @@ static void LogAccessExclusiveLocks(int nlocks, xl_standby_lock* locks) (void)XLogInsert(RM_STANDBY_ID, XLOG_STANDBY_LOCK); } +static void log_access_exclusive_locks_new(int nlocks, XlStandbyLockNew *locks) +{ + XLogStandbyLocksNew xlrec; + + xlrec.nlocks = nlocks; + + XLogBeginInsert(); + XLogRegisterData((char *)&xlrec, MIN_SIZE_OF_XACT_STANDBY_LOCKS_NEW); + XLogRegisterData((char *)locks, nlocks * sizeof(XlStandbyLockNew)); + + (void)XLogInsert(RM_STANDBY_ID, XLOG_STANDBY_LOCK | PARTITION_ACCESS_EXCLUSIVE_LOCK_UPGRADE_FLAG); +} + /* * Individual logging of AccessExclusiveLocks for use during LockAcquire() */ -void LogAccessExclusiveLock(Oid dbOid, Oid relOid) +void LogAccessExclusiveLock(Oid dbOid, Oid relOid, uint32 seq) { - if (ENABLE_DMS) { - return; - } - - xl_standby_lock xlrec; - - xlrec.xid = GetTopTransactionId(); + if (t_thrd.proc->workingVersionNum < PARTITION_ACCESS_EXCLUSIVE_LOCK_UPGRADE_VERSION) { + xl_standby_lock xlrec; + xlrec.xid = GetTopTransactionId(); + /* + * Decode the lock_tag back to the original values, to avoid sending lots + * of empty bytes with every message. See lock.h to check how a lock_tag + * is defined for LOCKTAG_RELATION + */ + xlrec.dbOid = dbOid; + xlrec.relOid = relOid; - /* - * Decode the locktag back to the original values, to avoid sending lots - * of empty bytes with every message. See lock.h to check how a locktag - * is defined for LOCKTAG_RELATION - */ - xlrec.dbOid = dbOid; - xlrec.relOid = relOid; + LogAccessExclusiveLocks(1, &xlrec); + } else { + XlStandbyLockNew xlrec; + xlrec.xid = GetTopTransactionId(); + /* + * Decode the lock_tag back to the original values, to avoid sending lots + * of empty bytes with every message. See lock.h to check how a lock_tag + * is defined for LOCKTAG_RELATION + */ + xlrec.dbOid = dbOid; + xlrec.relOid = relOid; + xlrec.seq = seq; - LogAccessExclusiveLocks(1, &xlrec); + log_access_exclusive_locks_new(1, &xlrec); + } } /* @@ -1111,41 +1163,6 @@ void LogAccessExclusiveLockPrepare(void) (void)GetTopTransactionId(); } -static void LogReleaseAccessExclusiveLocks(int nlocks, xl_standby_lock* locks) -{ - xl_standby_locks xlrec; - - xlrec.nlocks = nlocks; - - XLogBeginInsert(); - XLogRegisterData((char*)&xlrec, MinSizeOfXactStandbyLocks); - XLogRegisterData((char*)locks, nlocks * sizeof(xl_standby_lock)); - - (void)XLogInsert(RM_STANDBY_ID, XLOG_STANDBY_UNLOCK); -} - -void LogReleaseAccessExclusiveLock(TransactionId xid, Oid dbOid, Oid relOid) -{ - if (ENABLE_DMS) { - return; - } - - xl_standby_lock xlrec; - - xlrec.xid = xid; - - /* - * Decode the locktag back to the original values, to avoid sending lots - * of empty bytes with every message. See lock.h to check how a locktag - * is defined for LOCKTAG_RELATION - */ - xlrec.dbOid = dbOid; - xlrec.relOid = relOid; - - LogReleaseAccessExclusiveLocks(1, &xlrec); -} - - void StandbyXlogStartup(void) { t_thrd.xlog_cxt.committing_csn_list = NIL; diff --git a/src/gausskernel/storage/lmgr/lock.cpp b/src/gausskernel/storage/lmgr/lock.cpp index c0294edef..8c5f7282d 100644 --- a/src/gausskernel/storage/lmgr/lock.cpp +++ b/src/gausskernel/storage/lmgr/lock.cpp @@ -773,8 +773,9 @@ static LockAcquireResult LockAcquireExtendedXC(const LOCKTAG *locktag, LOCKMODE * * First we prepare to log, then after lock acquired we issue log record. */ - if (lockmode >= AccessExclusiveLock && (locktag->locktag_type == LOCKTAG_RELATION || - locktag->locktag_type == LOCKTAG_PARTITION || locktag->locktag_type == LOCKTAG_PARTITION_SEQUENCE) && + if (lockmode >= AccessExclusiveLock && + (locktag->locktag_type == LOCKTAG_RELATION || locktag->locktag_type == LOCKTAG_PARTITION || + locktag->locktag_type == LOCKTAG_PARTITION_SEQUENCE || locktag->locktag_type == LOCKTAG_OBJECT) && !RecoveryInProgress() && XLogStandbyInfoActive()) { LogAccessExclusiveLockPrepare(); log_lock = true; @@ -1058,7 +1059,12 @@ static LockAcquireResult LockAcquireExtendedXC(const LOCKTAG *locktag, LOCKMODE * lots of empty bytes with every message. See lock.h to check how a * locktag is defined for LOCKTAG_RELATION */ - LogAccessExclusiveLock(locktag->locktag_field1, locktag->locktag_field2); + uint32 seq = InvalidOid; + if (locktag->locktag_type == LOCKTAG_PARTITION || locktag->locktag_type == LOCKTAG_PARTITION_SEQUENCE || + locktag->locktag_type == LOCKTAG_OBJECT) { + seq = locktag->locktag_field3; + } + LogAccessExclusiveLock(locktag->locktag_field1, locktag->locktag_field2, seq); } instr_stmt_report_lock(LOCK_END, lockmode); @@ -2057,8 +2063,9 @@ void LockReleaseAll(LOCKMETHODID lockmethodid, bool allLocks) * released: it is always and only released when a toplevel transaction * ends. */ - if (lockmethodid == DEFAULT_LOCKMETHOD) + if ((lockmethodid == DEFAULT_LOCKMETHOD) && (t_thrd.role != PAGEREDO)) { VirtualXactLockTableCleanup(); + } numLockModes = lockMethodTable->numLockModes; @@ -3799,30 +3806,6 @@ void lock_twophase_recover(TransactionId xid, uint16 info, void *recdata, uint32 LWLockRelease(partitionLock); } -/* - * Re-acquire a lock belonging to a transaction that was prepared, when - * when starting up into hot standby mode. - */ -void lock_twophase_standby_recover(TransactionId xid, uint16 info, void *recdata, uint32 len) -{ - TwoPhaseLockRecord *rec = (TwoPhaseLockRecord *)recdata; - LOCKTAG *locktag = NULL; - LOCKMODE lockmode; - LOCKMETHODID lockmethodid; - - Assert(len == sizeof(TwoPhaseLockRecord)); - locktag = &rec->locktag; - lockmode = rec->lockmode; - lockmethodid = locktag->locktag_lockmethodid; - - CHECK_LOCKMETHODID(lockmethodid); - - if (lockmode == AccessExclusiveLock && locktag->locktag_type == LOCKTAG_RELATION) { - StandbyAcquireAccessExclusiveLock(xid, locktag->locktag_field1 /* dboid */, - locktag->locktag_field2 /* reloid */); - } -} - /* * 2PC processing routine for COMMIT PREPARED case. * diff --git a/src/gausskernel/storage/replication/walreceiver.cpp b/src/gausskernel/storage/replication/walreceiver.cpp index b3a7009bf..34865ebeb 100755 --- a/src/gausskernel/storage/replication/walreceiver.cpp +++ b/src/gausskernel/storage/replication/walreceiver.cpp @@ -163,6 +163,7 @@ const char *g_reserve_param[] = { #endif "enable_huge_pages", "huge_page_size" + "exrto_standby_read_opt" }; const int g_reserve_param_num = lengthof(g_reserve_param); diff --git a/src/gausskernel/storage/smgr/storage_exrto_file.cpp b/src/gausskernel/storage/smgr/storage_exrto_file.cpp index 46ee9e494..20f433d8a 100644 --- a/src/gausskernel/storage/smgr/storage_exrto_file.cpp +++ b/src/gausskernel/storage/smgr/storage_exrto_file.cpp @@ -180,6 +180,18 @@ static ExRTOFileState *exrto_open_file(SMgrRelation reln, ForkNumber forknum, Bl return state; } + +bool exrto_check_unlink_relfilenode(const RelFileNode rnode) +{ + HTAB *relfilenode_hashtbl = g_instance.bgwriter_cxt.unlink_rel_hashtbl; + bool found = false; + + LWLockAcquire(g_instance.bgwriter_cxt.rel_hashtbl_lock, LW_SHARED); + (void)hash_search(relfilenode_hashtbl, &(rnode), HASH_FIND, &found); + LWLockRelease(g_instance.bgwriter_cxt.rel_hashtbl_lock); + + return found; +} BlockNumber get_single_file_nblocks(SMgrRelation reln, ForkNumber forknum, const ExRTOFileState *state) { @@ -350,6 +362,7 @@ void exrto_extend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, c int nbytes; struct stat file_stat; char* filename; + ExtensionBehavior behavior; type = exrto_file_type(reln->smgr_rnode.node.spcNode); total_block_num = get_total_block_num(type, reln->smgr_rnode.node.relNode, blocknum); @@ -359,7 +372,16 @@ void exrto_extend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, c } seekpos = (off_t)BLCKSZ * (total_block_num % EXRTO_FILE_BLOCKS[type]); - state = exrto_open_file(reln, forknum, blocknum, EXTENSION_CREATE); + behavior = (type == BLOCK_INFO_META ? EXTENSION_RETURN_NULL : EXTENSION_CREATE); + state = exrto_open_file(reln, forknum, blocknum, behavior); + if (state == NULL) { + Assert(type == BLOCK_INFO_META); + if (exrto_check_unlink_relfilenode(reln->smgr_rnode.node)) { + return; + } else { + state = exrto_open_file(reln, forknum, blocknum, EXTENSION_CREATE); + } + } filename = FilePathName(state->file[forknum]); if (stat(filename, &file_stat) < 0) { char filepath[EXRTO_FILE_PATH_LEN]; @@ -380,12 +402,15 @@ void exrto_extend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, c (int)(EXRTO_FILE_SIZE[type] - file_stat.st_size)); nbytes = FilePWrite(state->file[forknum], NULL, extend_size, file_stat.st_size); if (nbytes != extend_size) { + char filepath[EXRTO_FILE_PATH_LEN]; + errno_t rc = strcpy_s(filepath, EXRTO_FILE_PATH_LEN, filename); + securec_check(rc, "\0", "\0"); exrto_close(reln, forknum, InvalidBlockNumber); if (nbytes < 0) { - ereport(ERROR, (errmsg("could not extend file \"%s\": %m.", filename))); + ereport(ERROR, (errmsg("could not extend file \"%s\": %m.", filepath))); } ereport(ERROR, - (errmsg("could not extend file \"%s\": wrote only %d of %d bytes.", filename, nbytes, extend_size))); + (errmsg("could not extend file \"%s\": wrote only %d of %d bytes.", filepath, nbytes, extend_size))); } Assert(get_single_file_nblocks(reln, forknum, state) <= ((BlockNumber)EXRTO_FILE_BLOCKS[type])); @@ -456,6 +481,7 @@ void exrto_write(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, co uint64 total_block_num; off_t seekpos; int nbytes; + ExtensionBehavior behavior; type = exrto_file_type(reln->smgr_rnode.node.spcNode); total_block_num = get_total_block_num(type, reln->smgr_rnode.node.relNode, blocknum); @@ -467,7 +493,16 @@ void exrto_write(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, co Assert(seekpos < (off_t)EXRTO_FILE_SIZE[type]); - state = exrto_open_file(reln, forknum, blocknum, EXTENSION_CREATE); + behavior = (type == BLOCK_INFO_META ? EXTENSION_RETURN_NULL : EXTENSION_CREATE); + state = exrto_open_file(reln, forknum, blocknum, behavior); + if (state == NULL) { + Assert(type == BLOCK_INFO_META); + if (exrto_check_unlink_relfilenode(reln->smgr_rnode.node)) { + return; + } else { + state = exrto_open_file(reln, forknum, blocknum, EXTENSION_CREATE); + } + } nbytes = FilePWrite(state->file[forknum], buffer, BLCKSZ, seekpos); if (nbytes != BLCKSZ) { char *filename = FilePathName(state->file[forknum]); @@ -547,13 +582,23 @@ void exrto_writeback(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum uint64 total_block_num; type = exrto_file_type(reln->smgr_rnode.node.spcNode); total_block_num = get_total_block_num(type, reln->smgr_rnode.node.relNode, blocknum); + ExtensionBehavior behavior = (type == BLOCK_INFO_META ? EXTENSION_RETURN_NULL : EXTENSION_CREATE); while (nblocks > 0) { BlockNumber nflush = nblocks; off_t seekpos; ExRTOFileState *state = NULL; uint64 segnum_start, segnum_end; - state = exrto_open_file(reln, forknum, blocknum, EXTENSION_CREATE); + state = exrto_open_file(reln, forknum, blocknum, behavior); + if (state == NULL) { + Assert(type == BLOCK_INFO_META); + /* only check at first time */ + if (exrto_check_unlink_relfilenode(reln->smgr_rnode.node)) { + return; + } else { + state = exrto_open_file(reln, forknum, blocknum, EXTENSION_CREATE); + } + } segnum_start = total_block_num / EXRTO_FILE_BLOCKS[type]; segnum_end = (total_block_num + nblocks - 1) / EXRTO_FILE_BLOCKS[type]; @@ -572,5 +617,6 @@ void exrto_writeback(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum Assert(((total_block_num + nflush) >> UINT64_HALF) == (total_block_num >> UINT64_HALF)); total_block_num += nflush; blocknum = (BlockNumber)total_block_num; + behavior = EXTENSION_CREATE; } } diff --git a/src/include/access/clog.h b/src/include/access/clog.h index c637bf48a..68ae0c8cd 100644 --- a/src/include/access/clog.h +++ b/src/include/access/clog.h @@ -37,6 +37,7 @@ #define TransactionIdToPgIndex(xid) ((xid) % (TransactionId)CLOG_XACTS_PER_PAGE) #define TransactionIdToByte(xid) (TransactionIdToPgIndex(xid) / CLOG_XACTS_PER_BYTE) #define TransactionIdToBIndex(xid) ((xid) % (TransactionId)CLOG_XACTS_PER_BYTE) +#define PAGE_TO_TRANSACTION_ID(pageno) ((pageno) * (TransactionId)CLOG_XACTS_PER_PAGE) #define CLogPageNoToStartXactId(pageno) ((pageno > 0)?((pageno -1) * CLOG_XACTS_PER_PAGE): 0) /* CLog lwlock partition*/ diff --git a/src/include/access/extreme_rto/page_redo.h b/src/include/access/extreme_rto/page_redo.h index 7d789f858..90674a455 100644 --- a/src/include/access/extreme_rto/page_redo.h +++ b/src/include/access/extreme_rto/page_redo.h @@ -204,6 +204,7 @@ void GetThreadNameIfPageRedoWorker(int argc, char *argv[], char **threadNamePtr) extern bool RedoWorkerIsUndoSpaceWorker(); uint32 GetMyPageRedoWorkerIdWithLock(); +void redo_worker_release_all_locks(); PGPROC *GetPageRedoWorkerProc(PageRedoWorker *worker); /* Worker main function. */ @@ -257,9 +258,6 @@ void BatchClearRecoveryThreadHashTbl(Oid spcNode, Oid dbNode); void RecordBadBlockAndPushToRemote(XLogBlockDataParse *datadecode, PageErrorType error_type, XLogRecPtr old_lsn, XLogPhyBlock pblk); void SeqCheckRemoteReadAndRepairPage(); -void exrto_generate_snapshot(XLogRecPtr trxn_lsn); -void exrto_read_snapshot(Snapshot snapshot); -XLogRecPtr exrto_calculate_recycle_position(bool force_recyle); -TransactionId exrto_calculate_recycle_xmin_for_undo(); +bool exceed_send_lsn_forworder_interval(); } // namespace extreme_rto #endif diff --git a/src/include/access/extreme_rto/standby_read/block_info_meta.h b/src/include/access/extreme_rto/standby_read/block_info_meta.h index d4cd7ed73..9a104db69 100644 --- a/src/include/access/extreme_rto/standby_read/block_info_meta.h +++ b/src/include/access/extreme_rto/standby_read/block_info_meta.h @@ -80,9 +80,13 @@ typedef enum { BlockMetaInfo *get_block_meta_info_by_relfilenode(const BufferTag &buf_tag, BufferAccessStrategy strategy, ReadBufferMode mode, Buffer *buffer, bool need_share_lock = false); void insert_lsn_to_block_info( - StandbyReadMetaInfo* mete_info, const BufferTag& buf_tag, const Page base_page, XLogRecPtr next_lsn); -StandbyReadRecyleState recyle_block_info( - const BufferTag& buf_tag, LsnInfoPosition base_page_info_pos, XLogRecPtr next_base_page_lsn, XLogRecPtr recyle_lsn); + StandbyReadMetaInfo *mete_info, const BufferTag &buf_tag, const Page base_page, XLogRecPtr next_lsn); +void insert_lsn_to_block_info_for_opt( + StandbyReadMetaInfo *mete_info, const BufferTag &buf_tag, const Page base_page, XLogRecPtr next_lsn); + +StandbyReadRecyleState recyle_block_info(const BufferTag &buf_tag, LsnInfoPosition base_page_info_pos, + XLogRecPtr next_base_page_lsn, XLogRecPtr recyle_lsn, + XLogRecPtr *block_info_max_lsn); bool get_page_lsn_info(const BufferTag& buf_tag, BufferAccessStrategy strategy, XLogRecPtr read_lsn, StandbyReadLsnInfoArray* lsn_info); static inline bool is_block_info_page_valid(BlockInfoPageHeader* header) diff --git a/src/include/access/extreme_rto/standby_read/lsn_info_meta.h b/src/include/access/extreme_rto/standby_read/lsn_info_meta.h index 7694bb984..8e641b5ab 100644 --- a/src/include/access/extreme_rto/standby_read/lsn_info_meta.h +++ b/src/include/access/extreme_rto/standby_read/lsn_info_meta.h @@ -146,6 +146,8 @@ bool is_base_page_map_bit_set(Page page, uint32 which_bit); void recycle_one_lsn_info_list(const BufferTag& buf_tag, LsnInfoPosition page_info_pos, XLogRecPtr recycle_lsn, LsnInfoPosition *min_page_info_pos, XLogRecPtr *min_lsn); void standby_read_recyle_per_workers(StandbyReadMetaInfo *standby_read_meta_info, XLogRecPtr recycle_lsn); +LsnInfoPosition get_nearest_base_page_pos( + const BufferTag &buf_tag, const LsnInfoDoubleList &lsn_info_list, XLogRecPtr read_lsn); } // namespace extreme_rto_standby_read #endif \ No newline at end of file diff --git a/src/include/access/extreme_rto/standby_read/standby_read_base.h b/src/include/access/extreme_rto/standby_read/standby_read_base.h index d61690a93..efea1de98 100644 --- a/src/include/access/extreme_rto/standby_read/standby_read_base.h +++ b/src/include/access/extreme_rto/standby_read/standby_read_base.h @@ -114,12 +114,18 @@ inline uint64 get_total_block_num(ExRTOFileType type, uint32 high, uint32 low) void exrto_clean_dir(void); void exrto_recycle_old_dir(void); void exrto_standby_read_init(); +void exrto_generate_snapshot(XLogRecPtr trxn_lsn); +void exrto_read_snapshot(Snapshot snapshot); +XLogRecPtr exrto_calculate_recycle_position(bool force_recyle); +TransactionId exrto_calculate_recycle_xmin_for_undo(); void buffer_drop_exrto_standby_read_buffers(StandbyReadMetaInfo *meta_info = NULL); void exrto_unlink_file_with_prefix(char *target_prefix, ExRTOFileType type, uint64 segno = 0); extern void XLogDumpDisplayRecord(XLogReaderState *record, char *strOutput); extern XLogRecPtr UpdateNextLSN(XLogRecPtr cur_lsn, XLogRecPtr end_lsn, XLogReaderState *xlogreader_state, bool *found); namespace extreme_rto_standby_read { void dump_error_all_info(const RelFileNode &rnode, ForkNumber forknum, BlockNumber blocknum); +Buffer standby_read_buf_new( + Relation reln, ForkNumber fork_num, BlockNumber block_num, ReadBufferMode mode, BufferAccessStrategy strategy); } #ifdef ENABLE_UT extern Page get_page_from_buffer(Buffer buf); diff --git a/src/include/access/extreme_rto/standby_read/standby_read_delay_ddl.h b/src/include/access/extreme_rto/standby_read/standby_read_delay_ddl.h index ed372174c..a2623640c 100644 --- a/src/include/access/extreme_rto/standby_read/standby_read_delay_ddl.h +++ b/src/include/access/extreme_rto/standby_read/standby_read_delay_ddl.h @@ -37,4 +37,5 @@ void init_delay_ddl_file(); void update_delay_ddl_db(Oid db_id, Oid tablespace_id, XLogRecPtr lsn); void update_delay_ddl_files(ColFileNode* xnodes, int nrels, XLogRecPtr lsn); void delete_by_table_space(Oid tablespace_id); +void update_delay_ddl_file_truncate_clog(XLogRecPtr lsn, int64 pageno); #endif \ No newline at end of file diff --git a/src/include/access/multi_redo_api.h b/src/include/access/multi_redo_api.h index 4ec8cee78..b3bda3fa7 100644 --- a/src/include/access/multi_redo_api.h +++ b/src/include/access/multi_redo_api.h @@ -67,8 +67,10 @@ static const uint32 PAGE_REDO_WORKER_EXIT = 3; static const uint32 BIG_RECORD_LENGTH = XLOG_BLCKSZ * 16; #define IS_EXRTO_READ (IsExtremeRedo() && g_instance.attr.attr_storage.EnableHotStandby && IsDefaultExtremeRtoMode()) -#define IS_EXRTO_STANDBY_READ (IS_EXRTO_READ && pm_state_is_hot_standby()) #define IS_EXRTO_RECOVERY_IN_PROGRESS (RecoveryInProgress() && IsExtremeRedo() && IsDefaultExtremeRtoMode()) +#define IS_EXRTO_STANDBY_READ (pm_state_is_hot_standby() && IS_EXRTO_READ) +#define IS_EXRTO_READ_OPT \ + (g_instance.attr.attr_storage.EnableHotStandby && g_instance.attr.attr_storage.enable_exrto_standby_read_opt) inline bool is_exrto_standby_read_worker() { diff --git a/src/include/access/nbtree.h b/src/include/access/nbtree.h index 55bf55969..86c611596 100644 --- a/src/include/access/nbtree.h +++ b/src/include/access/nbtree.h @@ -369,6 +369,10 @@ enum { BTREE_NEWROOT_META_BLOCK_NUM }; +enum { + BTREE_REUSE_PAGE_BLOCK_NUM = 0, +}; + typedef struct xl_btree_metadata_old { BlockNumber root; uint32 level; @@ -1334,9 +1338,10 @@ extern Buffer _bt_getroot(Relation rel, int access); extern Buffer _bt_gettrueroot(Relation rel); extern int _bt_getrootheight(Relation rel); extern void _bt_checkbuffer_valid(Relation rel, Buffer buf); -extern void _bt_checkpage(Relation rel, Buffer buf); +extern void _bt_checkpage(Relation rel, Buffer buf, BlockNumber par_blkno = InvalidBlockNumber); extern Buffer _bt_getbuf(Relation rel, BlockNumber blkno, int access); -extern Buffer _bt_relandgetbuf(Relation rel, Buffer obuf, BlockNumber blkno, int access); +extern Buffer _bt_relandgetbuf(Relation rel, Buffer obuf, BlockNumber blkno, int access, + BlockNumber par_blkno = InvalidBlockNumber); extern void _bt_relbuf(Relation rel, Buffer buf); extern void _bt_pageinit(Page page, Size size); extern bool _bt_page_recyclable(Page page); diff --git a/src/include/access/parallel_recovery/page_redo.h b/src/include/access/parallel_recovery/page_redo.h index e3056d9fc..22dbfa51a 100644 --- a/src/include/access/parallel_recovery/page_redo.h +++ b/src/include/access/parallel_recovery/page_redo.h @@ -183,7 +183,6 @@ struct PageRedoWorker { uint64 statWaitReplay; pg_atomic_uint32 readyStatus; MemoryContext oldCtx; - int bufferPinWaitBufId; RedoTimeCost timeCostList[TIME_COST_NUM]; uint32 remoteReadPageNum; HTAB *badPageHashTbl; diff --git a/src/include/access/twophase_rmgr.h b/src/include/access/twophase_rmgr.h index 7f461b5f5..7a59fc6d5 100644 --- a/src/include/access/twophase_rmgr.h +++ b/src/include/access/twophase_rmgr.h @@ -30,7 +30,6 @@ typedef uint8 TwoPhaseRmgrId; extern const TwoPhaseCallback g_twophase_recover_callbacks[]; extern const TwoPhaseCallback g_twophase_postcommit_callbacks[]; extern const TwoPhaseCallback g_twophase_postabort_callbacks[]; -extern const TwoPhaseCallback g_twophase_standby_recover_callbacks[]; extern void RegisterTwoPhaseRecord(TwoPhaseRmgrId rmid, uint16 info, const void* data, uint32 len); diff --git a/src/include/access/ustore/knl_uvisibility.h b/src/include/access/ustore/knl_uvisibility.h index 34d52c63c..eef4711e3 100644 --- a/src/include/access/ustore/knl_uvisibility.h +++ b/src/include/access/ustore/knl_uvisibility.h @@ -16,6 +16,8 @@ #ifndef KNL_UVISIBILITY_H #define KNL_UVISIBILITY_H +#include "postmaster/postmaster.h" + typedef struct UHeapTupleTransInfo { int td_slot; TransactionId xid; @@ -87,4 +89,28 @@ bool UHeapTupleHasSerializableConflictOut(bool visible, Relation relation, ItemP void UHeapTupleCheckVisible(Snapshot snapshot, UHeapTuple tuple, Buffer buffer); void UHeapUpdateTDInfo(int tdSlot, Buffer buffer, OffsetNumber offnum, UHeapTupleTransInfo* uinfo); + +inline bool TransactionIdOlderThanAllUndo(TransactionId xid) +{ + /* to slove standby read consistency problem */ + if (RecoveryInProgress()) { + uint64 standby_recycle_xid = pg_atomic_read_u64(&g_instance.undo_cxt.hotStandbyRecycleXid); + return xid < standby_recycle_xid; + } + + uint64 cutoff = pg_atomic_read_u64(&g_instance.undo_cxt.globalRecycleXid); + return xid < cutoff; +} + +inline bool TransactionIdOlderThanFrozenXid(TransactionId xid) +{ + /* to slove standby read consistency problem */ + if (RecoveryInProgress()) { + uint64 standby_recycle_xid = pg_atomic_read_u64(&g_instance.undo_cxt.hotStandbyRecycleXid); + return xid < standby_recycle_xid; + } + + uint64 cutoff = pg_atomic_read_u64(&g_instance.undo_cxt.globalFrozenXid); + return xid < cutoff; +} #endif diff --git a/src/include/access/xact.h b/src/include/access/xact.h index 09ad0c69a..76af9756f 100644 --- a/src/include/access/xact.h +++ b/src/include/access/xact.h @@ -565,6 +565,6 @@ extern void BeginTxnForAutoCommitOff(); extern void SetTxnInfoForSSLibpqsw(TransactionId xid, CommandId cid); extern void ClearTxnInfoForSSLibpqsw(); extern bool IsTransactionInProgressState(); -extern void unlink_relfiles(_in_ ColFileNode *xnodes, _in_ int nrels); +extern void unlink_relfiles(_in_ ColFileNode *xnodes, _in_ int nrels, bool is_old_delay_ddl = false); void xact_redo_log_drop_segs(_in_ ColFileNode *xnodes, _in_ int nrels, XLogRecPtr lsn); #endif /* XACT_H */ diff --git a/src/include/access/xlogproc.h b/src/include/access/xlogproc.h index bed5a495f..d9adbdddc 100755 --- a/src/include/access/xlogproc.h +++ b/src/include/access/xlogproc.h @@ -160,6 +160,7 @@ typedef enum { BLOCK_DATA_SEG_SPACE_SHRINK, BLOCK_DATA_SEG_FULL_SYNC_TYPE, BLOCK_DATA_SEG_EXTEND, + BLOCK_DATA_CLEANUP_TYPE, } XLogBlockParseEnum; /* ********BLOCK COMMON HEADER END ***************** */ @@ -498,6 +499,7 @@ typedef struct { Oid dbNode; /* database */ int2 bucketNode; /* bucket */ uint2 opt; + bool is_conflict_type; /* whether wal log type is conflict with standby read if redo */ XLogPhyBlock pblk; } XLogBlockHead; @@ -594,6 +596,10 @@ typedef struct { Size dataLen; } XLogBlockSegNewPage; +typedef struct { + TransactionId removed_xid; +} WalCleanupInfoParse; + typedef struct { XLogBlockHead blockhead; XLogBlockRedoHead redohead; @@ -624,6 +630,7 @@ typedef struct { XLogBlockSegDdlParse blocksegddlrec; XLogBlockSegFullSyncParse blocksegfullsyncrec; XLogBlockSegNewPage blocksegnewpageinfo; + WalCleanupInfoParse clean_up_info; } extra_rec; } XLogBlockParse; @@ -1113,7 +1120,7 @@ void XLogRecSetSegNewPageInfo(XLogBlockSegNewPage *state, char *mainData, Size l void XLogRecSetAuxiBlkNumState(XLogBlockDataParse* blockdatarec, BlockNumber auxilaryblkn1, BlockNumber auxilaryblkn2); void XLogRecSetBlockDataStateContent(XLogReaderState *record, uint32 blockid, XLogBlockDataParse *blockdatarec); void XLogRecSetBlockDataState(XLogReaderState* record, uint32 blockid, XLogRecParseState* recordblockstate, - XLogBlockParseEnum type = BLOCK_DATA_MAIN_DATA_TYPE); + XLogBlockParseEnum type = BLOCK_DATA_MAIN_DATA_TYPE, bool is_conflict_type = false); extern char* XLogBlockDataGetBlockData(XLogBlockDataParse* datadecode, Size* len); void Heap2RedoDataBlock(XLogBlockHead* blockhead, XLogBlockDataParse* blockdatarec, RedoBufferInfo* bufferinfo); extern void HeapRedoDataBlock( @@ -1124,6 +1131,7 @@ extern void xlog_redo_data_block( extern void XLogRecSetBlockDdlState(XLogBlockDdlParse* blockddlstate, uint32 blockddltype, char *mainData, int rels = 1, bool compress = false, uint32 main_data_len = 0); XLogRedoAction XLogCheckBlockDataRedoAction(XLogBlockDataParse* datadecode, RedoBufferInfo* bufferinfo); +extern void wal_rec_set_clean_up_info_state(WalCleanupInfoParse *parse_state, TransactionId removed_xid); void BtreeRedoDataBlock(XLogBlockHead* blockhead, XLogBlockDataParse* blockdatarec, RedoBufferInfo* bufferinfo); void Btree2RedoDataBlock(XLogBlockHead* blockhead, XLogBlockDataParse* blockdatarec, RedoBufferInfo* bufferinfo); diff --git a/src/include/knl/knl_guc/knl_instance_attr_storage.h b/src/include/knl/knl_guc/knl_instance_attr_storage.h index 89ff5a3af..f32e80dde 100755 --- a/src/include/knl/knl_guc/knl_instance_attr_storage.h +++ b/src/include/knl/knl_guc/knl_instance_attr_storage.h @@ -210,12 +210,11 @@ typedef struct knl_instance_attr_storage { int max_active_gtt; /* extreme-rto standby read */ - int64 max_standby_base_page_size; - int64 max_standby_lsn_info_size; int base_page_saved_interval; double standby_force_recycle_ratio; int standby_recycle_interval; int standby_max_query_time; + bool enable_exrto_standby_read_opt; #ifndef ENABLE_MULTIPLE_NODES bool enable_save_confirmed_lsn; #endif diff --git a/src/include/knl/knl_guc/knl_session_attr_storage.h b/src/include/knl/knl_guc/knl_session_attr_storage.h index af7a7e430..75c0e34ed 100755 --- a/src/include/knl/knl_guc/knl_session_attr_storage.h +++ b/src/include/knl/knl_guc/knl_session_attr_storage.h @@ -267,6 +267,10 @@ typedef struct knl_session_attr_storage { int ignore_standby_lsn_window; int ignore_feedback_xmin_window; int subscription_conflict_resolution; + + /* extreme-rto standby read */ + int64 max_standby_base_page_size; + int64 max_standby_lsn_info_size; } knl_session_attr_storage; #endif /* SRC_INCLUDE_KNL_KNL_SESSION_ATTR_STORAGE */ diff --git a/src/include/knl/knl_instance.h b/src/include/knl/knl_instance.h index 41fdcd7f1..2aaa2f461 100755 --- a/src/include/knl/knl_instance.h +++ b/src/include/knl/knl_instance.h @@ -756,11 +756,16 @@ typedef struct knl_g_parallel_redo_context { char* ali_buf; XLogRedoNumStatics xlogStatics[RM_NEXT_ID][MAX_XLOG_INFO_NUM]; RedoCpuBindControl redoCpuBindcontrl; - XLogRecPtr global_recycle_lsn; /* extreme-rto standby read */ HTAB **redoItemHash; /* used in ondemand extreme RTO */ + /* extreme-rto standby read */ + TransactionId exrto_recyle_xmin; + XLogRecPtr global_recycle_lsn; ExrtoSnapshot exrto_snapshot; + TimestampTz exrto_send_lsn_forworder_time; StandbyReadDelayDdlState standby_read_delay_ddl_stat; uint64 max_clog_pageno; + int *buffer_pin_wait_buf_ids; + int buffer_pin_wait_buf_len; } knl_g_parallel_redo_context; typedef struct knl_g_heartbeat_context { @@ -931,6 +936,7 @@ typedef struct knl_g_undo_context { pg_atomic_uint64 globalFrozenXid; /* Oldest transaction id which is having undo. */ pg_atomic_uint64 globalRecycleXid; + pg_atomic_uint64 hotStandbyRecycleXid; bool is_exrto_residual_undo_file_recycled; } knl_g_undo_context; diff --git a/src/include/knl/knl_thread.h b/src/include/knl/knl_thread.h index 6b1c7b11f..b3d5cf3a9 100755 --- a/src/include/knl/knl_thread.h +++ b/src/include/knl/knl_thread.h @@ -1556,6 +1556,7 @@ typedef struct knl_t_undorecycler_context { /* Flags set by signal handlers */ volatile sig_atomic_t got_SIGHUP; volatile sig_atomic_t shutdown_requested; + bool is_recovery_in_progress; } knl_t_undorecycler_context; typedef struct knl_t_rollback_requests_context { diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h index abe48258f..a52ef3218 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h @@ -136,6 +136,7 @@ extern const uint32 SLOW_SQL_VERSION_NUM; extern const uint32 INDEX_HINT_VERSION_NUM; extern const uint32 CREATE_TABLE_AS_VERSION_NUM; extern const uint32 GB18030_2022_VERSION_NUM; +extern const uint32 PARTITION_ACCESS_EXCLUSIVE_LOCK_UPGRADE_VERSION; extern void register_backend_version(uint32 backend_version); extern bool contain_backend_version(uint32 version_number); diff --git a/src/include/storage/lock/lock.h b/src/include/storage/lock/lock.h index a67445a41..8bebfefaa 100644 --- a/src/include/storage/lock/lock.h +++ b/src/include/storage/lock/lock.h @@ -675,13 +675,19 @@ typedef struct xl_standby_lock { Oid relOid; } xl_standby_lock; +typedef struct XlStandbyLockNew { + TransactionId xid; /* xid of holder of ACCESS_EXCLUSIVE_LOCK */ + Oid dbOid; + Oid relOid; + uint32 seq; +} XlStandbyLockNew; + extern xl_standby_lock* GetRunningTransactionLocks(int* nlocks); extern const char* GetLockmodeName(LOCKMETHODID lockmethodid, LOCKMODE mode); extern void lock_twophase_recover(TransactionId xid, uint16 info, void* recdata, uint32 len); extern void lock_twophase_postcommit(TransactionId xid, uint16 info, void* recdata, uint32 len); extern void lock_twophase_postabort(TransactionId xid, uint16 info, void* recdata, uint32 len); -extern void lock_twophase_standby_recover(TransactionId xid, uint16 info, void* recdata, uint32 len); extern DeadLockState DeadLockCheck(PGPROC* proc); extern PGPROC* GetBlockingAutoVacuumPgproc(void); diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h index 915f3f791..085861b3d 100755 --- a/src/include/storage/proc.h +++ b/src/include/storage/proc.h @@ -566,17 +566,6 @@ void CancelBlockedRedistWorker(LOCK* lock, LOCKMODE lockmode); extern void BecomeLockGroupLeader(void); extern void BecomeLockGroupMember(PGPROC *leader); -static inline bool TransactionIdOlderThanAllUndo(TransactionId xid) -{ - uint64 cutoff = pg_atomic_read_u64(&g_instance.undo_cxt.globalRecycleXid); - return xid < cutoff; -} -static inline bool TransactionIdOlderThanFrozenXid(TransactionId xid) -{ - uint64 cutoff = pg_atomic_read_u64(&g_instance.undo_cxt.globalFrozenXid); - return xid < cutoff; -} - extern int GetThreadPoolStreamProcNum(void); #endif /* PROC_H */ diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h index 6f0b4aa9d..164018c89 100755 --- a/src/include/storage/procarray.h +++ b/src/include/storage/procarray.h @@ -116,8 +116,8 @@ extern VirtualTransactionId *GetConflictingVirtualXIDs(TransactionId limitXmin, CommitSeqNo limitXminCSN = InvalidCommitSeqNo, TransactionId* xminArray = NULL); extern ThreadId CancelVirtualTransaction(const VirtualTransactionId& vxid, ProcSignalReason sigmode); -extern bool proc_array_cancel_conflicting_proc(TransactionId latest_removed_xid, - bool reach_max_check_times); +extern bool proc_array_cancel_conflicting_proc( + TransactionId latest_removed_xid, XLogRecPtr truncate_redo_lsn, bool reach_max_check_times); extern bool MinimumActiveBackends(int min); extern int CountDBBackends(Oid database_oid); @@ -141,6 +141,7 @@ extern void ProcArrayGetReplicationSlotXmin(TransactionId* xmin, TransactionId* extern TransactionId GetGlobal2pcXmin(); extern void CSNLogRecordAssignedTransactionId(TransactionId newXid); +extern void UpdateCleanUpInfo(TransactionId limitXmin, XLogRecPtr lsn); /* * Fast search of ProcArray mapping (xid => proc array index), diff --git a/src/include/storage/standby.h b/src/include/storage/standby.h index 840043baf..5de989ba8 100755 --- a/src/include/storage/standby.h +++ b/src/include/storage/standby.h @@ -41,7 +41,7 @@ extern void CheckRecoveryConflictDeadlock(void); * to make hot standby work. That includes logging AccessExclusiveLocks taken * by transactions and running-xacts snapshots. */ -extern void StandbyAcquireAccessExclusiveLock(TransactionId xid, Oid dbOid, Oid relOid); +extern void StandbyAcquireAccessExclusiveLock(TransactionId xid, Oid dbOid, Oid relOid, uint32 seq); extern void StandbyReleaseLockTree(TransactionId xid, int nsubxids, TransactionId* subxids); extern void StandbyReleaseAllLocks(void); extern void StandbyReleaseOldLocks(TransactionId oldestRunningXid); @@ -60,12 +60,18 @@ extern bool standbyWillTouchStandbyLocks(XLogReaderState* record); #define XLOG_STANDBY_CSN_COMMITTING 0x40 #define XLOG_STANDBY_CSN_ABORTED 0x50 +#define PARTITION_ACCESS_EXCLUSIVE_LOCK_UPGRADE_FLAG 0x01 typedef struct xl_standby_locks { int nlocks; /* number of entries in locks array */ xl_standby_lock locks[FLEXIBLE_ARRAY_MEMBER]; /* VARIABLE LENGTH ARRAY */ } xl_standby_locks; +typedef struct XLogStandbyLocksNew { + int nlocks; /* number of entries in locks array */ + XlStandbyLockNew locks[FLEXIBLE_ARRAY_MEMBER]; /* VARIABLE LENGTH ARRAY */ +} XLogStandbyLocksNew; + /* * Keep track of all the locks owned by a given transaction. */ @@ -77,6 +83,7 @@ typedef struct RecoveryLockListsEntry #define MinSizeOfXactStandbyLocks offsetof(xl_standby_locks, locks) +#define MIN_SIZE_OF_XACT_STANDBY_LOCKS_NEW offsetof(XLogStandbyLocksNew, locks) /* * When we write running xact data to WAL, we use this structure. @@ -140,9 +147,8 @@ typedef struct RunningTransactionsData { typedef RunningTransactionsData* RunningTransactions; -extern void LogAccessExclusiveLock(Oid dbOid, Oid relOid); +extern void LogAccessExclusiveLock(Oid dbOid, Oid relOid, uint32 seq); extern void LogAccessExclusiveLockPrepare(void); -extern void LogReleaseAccessExclusiveLock(TransactionId xid, Oid dbOid, Oid relOid); extern XLogRecPtr LogStandbySnapshot(void); #endif /* STANDBY_H */ diff --git a/src/include/utils/guc.h b/src/include/utils/guc.h index 6f1b55706..ff38a9ca2 100755 --- a/src/include/utils/guc.h +++ b/src/include/utils/guc.h @@ -278,7 +278,7 @@ extern char* GetGucName(const char *command, char *target_guc_name); extern void BeginReportingGUCOptions(void); extern void ParseLongOption(const char* string, char** name, char** value); extern bool parse_int(const char* value, int* result, int flags, const char** hintmsg); -extern bool parse_int64(const char* value, int64* result, const char** hintmsg); +extern bool parse_int64(const char* value, int64* result, int flags, const char** hintmsg); extern bool parse_real(const char* value, double* result, int flags = 0, const char** hintmsg = NULL); double TimeUnitConvert(char** endptr, double value, int flags, const char** hintmsg); double MemoryUnitConvert(char** endptr, double value, int flags, const char** hintmsg); diff --git a/src/test/ha/standby_env.sh b/src/test/ha/standby_env.sh index 05743bfe7..b526dad79 100644 --- a/src/test/ha/standby_env.sh +++ b/src/test/ha/standby_env.sh @@ -1,7 +1,7 @@ #!/bin/sh #some enviroment vars -export g_base_port=25632 +export g_base_port=8888 export prefix=${GAUSSHOME} export g_pooler_base_port=`expr $g_base_port \+ 410` export g_base_standby_port=`expr $g_base_port \+ 400` @@ -11,28 +11,13 @@ export LD_LIBRARY_PATH=$prefix/lib:$prefix/lib/libobs:$LD_LIBRARY_PATH export PATH="$prefix/bin":$PATH export g_data_path="$install_path/hadata" -eth0ip=`/sbin/ifconfig eth0|sed -n 2p|awk '{ print $2 }'` -eth1ip=`/sbin/ifconfig eth1|sed -n 2p|awk '{ print $2 }'` -ethens=`/sbin/ifconfig ens4f0|sed -n 2p |awk '{ print $2 }'` -enp2s0f0=`/sbin/ifconfig enp2s0f0|sed -n 2p |awk '{ print $2 }'` -enp2s0f1=`/sbin/ifconfig enp2s0f1|sed -n 2p |awk '{ print $2 }'` -enp125s0f0=`/sbin/ifconfig enp125s0f0|sed -n 2p |awk '{ print $2 }'` +eth0ip=`/sbin/ifconfig | grep 'inet ' | grep -v 127.0.0.1 | head -1 | awk '{ print $2 }'` if [ -n "$eth0ip" ]; then - export eth_local_ip=$eth0ip -elif [ -n "$eth1ip" ];then - export eth_local_ip=$eth1ip -elif [ -n "$ethens" ];then - export eth_local_ip=$eth1ip -elif [ -n "$enp2s0f0" ];then - export eth_local_ip=$enp2s0f0 -elif [ -n "$enp2s0f1" ];then - export eth_local_ip=$enp2s0f1 -elif [ -n "$enp125s0f0" ];then - export eth_local_ip=$enp125s0f0 + export eth_local_ip=$eth0ip else - echo "error eth0 and eth1 not configured,exit" - exit 1 + echo "error eth0 and eth1 not configured,exit" + exit 1 fi export g_local_ip="127.0.0.1" diff --git a/src/test/ha/testcase/exrtostandbyread/single_standby_read_base.sh b/src/test/ha/testcase/exrtostandbyread/single_standby_read_base.sh index 56fcd6bde..b39d8e6f6 100644 --- a/src/test/ha/testcase/exrtostandbyread/single_standby_read_base.sh +++ b/src/test/ha/testcase/exrtostandbyread/single_standby_read_base.sh @@ -12,11 +12,29 @@ function check_select_result() fi } +function start_standby_read_cluster() +{ + start_primary_as_primary + start_standby +} + +function stop_standby_read_cluster() +{ + stop_primary + stop_standby +} + +function restart_standby_read_cluster() +{ + stop_standby_read_cluster + start_standby_read_cluster +} + function test_base_sql_func() { gsql -d test_standby_read_base -p $dn1_primary_port -c "DROP TABLE if exists test1; CREATE TABLE test1(contentId VARCHAR(128) NOT NULL, commentId VARCHAR(128) NOT NULL, appId VARCHAR(128) NOT NULL, PRIMARY KEY (contentId, commentId)) with(parallel_workers=8,storage_type=aSTORE);" gsql -d test_standby_read_base -p $dn1_primary_port -c "DROP TABLE if exists test2; CREATE TABLE test2(contentId VARCHAR(128) NOT NULL, commentId VARCHAR(128) NOT NULL, appId VARCHAR(128) NOT NULL, PRIMARY KEY (contentId, commentId)) with(storage_type=aSTORE,fillfactor=80) partition by hash(contentId);" - gsql -d test_standby_read_base -p $dn1_primary_port -c "DROP TABLE if exists test3; CREATE TABLE test3(contentId VARCHAR(128) NOT NULL, commentId VARCHAR(128) NOT NULL, appId VARCHAR(128) NOT NULL, PRIMARY KEY (contentId, commentId)) with(storage_type=uSTORE,fillfactor=40) partition by list(contentId) (partition p1 values ('1') ,partition p2 values ('2') ,partition p3 values ('3') ,partition p4 values (default));" + gsql -d test_standby_read_base -p $dn1_primary_port -c "DROP TABLE if exists test3; CREATE TABLE test3(contentId VARCHAR(128) NOT NULL, commentId VARCHAR(128) NOT NULL, appId VARCHAR(128) NOT NULL, PRIMARY KEY (contentId, commentId)) with(storage_type=aSTORE,fillfactor=40) partition by list(contentId) (partition p1 values ('1') ,partition p2 values ('2') ,partition p3 values ('3') ,partition p4 values (default));" gsql -d test_standby_read_base -p $dn1_primary_port -c "insert into test1 select generate_series(1,20), generate_series(1,20), generate_series(1,20);" gsql -d test_standby_read_base -p $dn1_primary_port -c "insert into test2 select generate_series(1,300), generate_series(1,300), generate_series(1,300);" @@ -77,7 +95,7 @@ function test_standby_read_base_func() gs_guc set -Z datanode -D $standby_data_dir -c "recovery_redo_workers = 1" gs_guc set -Z datanode -D $standby_data_dir -c "hot_standby = on" - start_cluster + start_standby_read_cluster echo "start cluster success" sleep 2 @@ -98,8 +116,7 @@ function test_standby_read_base_func() gs_guc set -Z datanode -D $primary_data_dir -c " recovery_max_workers = 4" gs_guc set -Z datanode -D $standby_data_dir -c " recovery_max_workers = 4" - kill_cluster - start_cluster + restart_standby_read_cluster test_base_sql_func @@ -113,8 +130,7 @@ function test_standby_read_base_func() gs_guc set -Z datanode -D $primary_data_dir -c " recovery_redo_workers = 4" gs_guc set -Z datanode -D $standby_data_dir -c " recovery_redo_workers = 4" - kill_cluster - start_cluster + restart_standby_read_cluster test_base_sql_func } From f7209fe990b49fb4cd08c4f186595dbd8624b19a Mon Sep 17 00:00:00 2001 From: hwhbj Date: Thu, 21 Sep 2023 16:14:11 +0800 Subject: [PATCH 270/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E8=BD=BB=E9=87=8F?= =?UTF-8?q?=E7=89=88=E5=9C=A8EULEROS=E5=AE=89=E8=A3=85=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- liteom/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/liteom/install.sh b/liteom/install.sh index b07fa22c0..e0987fdec 100644 --- a/liteom/install.sh +++ b/liteom/install.sh @@ -557,7 +557,7 @@ function decompress() kernel=$(cat /etc/euleros-release | awk -F ' ' '{print $1}' | tr a-z A-Z) if [ "${kernel}" = "Euleros" ] then - kernel="Euler" + kernel="openEuler" fi elif [ -f "/etc/openEuler-release" ] then From 9d2e08dd733fa75053919182a9a916499c813fcf Mon Sep 17 00:00:00 2001 From: z00793368 Date: Thu, 21 Sep 2023 17:06:32 +0800 Subject: [PATCH 271/304] fig bio_pub_key memory leak --- .../security/keymgr/encrypt/security_sm2_enc_key.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/gausskernel/security/keymgr/encrypt/security_sm2_enc_key.cpp b/src/gausskernel/security/keymgr/encrypt/security_sm2_enc_key.cpp index a1c1f27a3..01e9b59c9 100644 --- a/src/gausskernel/security/keymgr/encrypt/security_sm2_enc_key.cpp +++ b/src/gausskernel/security/keymgr/encrypt/security_sm2_enc_key.cpp @@ -98,6 +98,8 @@ Sm2KeyPair* generate_encrypt_pair_key() BIO_read(bio_pub_key, pub_key->ustr_val, (int)pub_key_len); priv_key->ustr_len = pri_key_len; pub_key->ustr_len = pub_key_len; + BIO_free(bio_priv_key); + BIO_free(bio_pub_key); sm2_key_pair = (Sm2KeyPair *)km_alloc(sizeof(Sm2KeyPair)); if (sm2_key_pair == NULL) { From 894aa9b83361f0a9ad662cf6f595b0e562a42ded Mon Sep 17 00:00:00 2001 From: movead Date: Thu, 21 Sep 2023 17:39:14 +0800 Subject: [PATCH 272/304] turn enable_batch_dispatch off by default --- src/common/backend/utils/misc/guc/guc_storage.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/backend/utils/misc/guc/guc_storage.cpp b/src/common/backend/utils/misc/guc/guc_storage.cpp index 1ee3c7c7f..5f221d528 100755 --- a/src/common/backend/utils/misc/guc/guc_storage.cpp +++ b/src/common/backend/utils/misc/guc/guc_storage.cpp @@ -1247,7 +1247,7 @@ static void InitStorageConfigureNamesBool() gettext_noop("Enable batch dispatch for parallel reovery"), NULL}, &g_instance.attr.attr_storage.enable_batch_dispatch, - true, + false, NULL, NULL, NULL}, From dc84164896dce32e721e17e9bc4e8c4f45efec3f Mon Sep 17 00:00:00 2001 From: li-qinlang Date: Thu, 21 Sep 2023 17:46:06 +0800 Subject: [PATCH 273/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=97=A0attach?= =?UTF-8?q?=E5=8F=AF=E8=BF=9B=E5=85=A5=E8=B0=83=E8=AF=95=E6=B5=81=E7=A8=8B?= =?UTF-8?q?&=E6=8A=A5=E9=94=99=E4=BF=A1=E6=81=AF=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/pl/plpgsql/src/pl_debugger.cpp | 4 +- src/common/pl/plpgsql/src/pl_debugger_api.cpp | 26 ++++++++----- src/include/utils/plpgsql.h | 2 +- .../regress/expected/pl_debugger_client.out | 37 +++++++++++++++---- .../regress/expected/pl_debugger_server.out | 19 ++++++++-- src/test/regress/sql/pl_debugger_client.sql | 7 +++- src/test/regress/sql/pl_debugger_server.sql | 17 +++++++++ 7 files changed, 87 insertions(+), 25 deletions(-) diff --git a/src/common/pl/plpgsql/src/pl_debugger.cpp b/src/common/pl/plpgsql/src/pl_debugger.cpp index e931fc454..c9da25752 100644 --- a/src/common/pl/plpgsql/src/pl_debugger.cpp +++ b/src/common/pl/plpgsql/src/pl_debugger.cpp @@ -743,11 +743,11 @@ void clean_up_debug_client(bool hasError) DebugClientInfo* client = u_sess->plsql_cxt.debug_client; /* clean comm idx*/ if (client->comm_idx < PG_MAX_DEBUG_CONN && client->comm_idx >= 0) { + uint64 clientSessionId = ENABLE_THREAD_POOL ? u_sess->session_id : t_thrd.proc_cxt.MyProcPid; PlDebuggerComm* debug_comm = &g_instance.pldebug_cxt.debug_comm[client->comm_idx]; AutoMutexLock debuglock(&debug_comm->mutex); debuglock.lock(); - if (debug_comm->hasClient() && - (debug_comm->clientId == u_sess->session_id || debug_comm->clientId == t_thrd.proc_cxt.MyProcPid)) { + if (debug_comm->hasClient() && debug_comm->clientId == clientSessionId) { /* only wake up server for error when it's not recevied server error */ if (hasError && debug_comm->IsServerWaited && !debug_comm->hasServerErrorOccured) { debug_comm->hasClientErrorOccured = true; diff --git a/src/common/pl/plpgsql/src/pl_debugger_api.cpp b/src/common/pl/plpgsql/src/pl_debugger_api.cpp index 8e7c0d5ad..9fe1a0727 100644 --- a/src/common/pl/plpgsql/src/pl_debugger_api.cpp +++ b/src/common/pl/plpgsql/src/pl_debugger_api.cpp @@ -331,8 +331,14 @@ static Datum get_info_local_data(const char* var_name, const int frameno, Functi */ Datum debug_client_info_code(PG_FUNCTION_ARGS) { - InterfaceCheck("info_code", false); Oid funcid = PG_GETARG_OID(0); + /* + * Anonymous block debugging call info_code() needs to get + * information from the server, so set needAttach to TRUE + * Procedure and function get information from system table, + * info_code() can be called at any time, so set needAttach to FALSE. + */ + InterfaceCheck("info_code", !OidIsValid(funcid)); const int DEBUG_LOCAL_VAR_TUPLE_ATTR_NUM = 3; @@ -493,7 +499,7 @@ Datum debug_client_add_breakpoint(PG_FUNCTION_ARGS) lines = debug_show_code_worker(funcOid, &nLine, &headerlines); if (lineno < 1 || (uint32)lineno > nLine - headerlines) { ereport(WARNING, (errcode(ERRCODE_WARNING), - errmsg("lineno must be within the range of [1, MaxLineNumber]" + errmsg("lineno must be within the range of [1, MaxLineNumber]." " Please use dbe_pldebugger.info_code for valid breakpoint candidates"))); PG_RETURN_INT32(-1); } @@ -523,7 +529,7 @@ Datum debug_client_add_breakpoint(PG_FUNCTION_ARGS) PG_RETURN_INT32(-1); } else if (ans == ADD_BP_ERR_OUT_OF_RANGE) { ereport(WARNING, (errcode(ERRCODE_WARNING), - errmsg("lineno must be within the range of [1, MaxLineNumber]" + errmsg("lineno must be within the range of [1, MaxLineNumber]." " Please use dbe_pldebugger.info_code for valid breakpoint candidates"))); PG_RETURN_INT32(-1); } else if (ans == ADD_BP_ERR_INVALID_BP_POS) { @@ -564,7 +570,7 @@ Datum debug_client_delete_breakpoint(PG_FUNCTION_ARGS) error: ereport(ERROR, (errmodule(MOD_PLDEBUGGER), errcode(ERRCODE_AMBIGUOUS_PARAMETER), - errmsg("invalid break point index"), + errmsg("invalid breakpoint index"), errdetail("the given index is either outside the range or already deleted"), errcause("try to delete a breakpoint that's never added"), erraction("use dbe_pldebugger.info_breakpoints() to show all valid breakpoints"))); @@ -600,7 +606,7 @@ Datum debug_client_enable_breakpoint(PG_FUNCTION_ARGS) error: ereport(ERROR, (errmodule(MOD_PLDEBUGGER), errcode(ERRCODE_AMBIGUOUS_PARAMETER), - errmsg("invalid break point index"), + errmsg("invalid breakpoint index"), errdetail("the given index is either outside the range or already enabled"), errcause("try to enable a breakpoint that's already enabled"), erraction("use dbe_pldebugger.info_breakpoints() to show all breakpoints"))); @@ -636,7 +642,7 @@ Datum debug_client_disable_breakpoint(PG_FUNCTION_ARGS) error: ereport(ERROR, (errmodule(MOD_PLDEBUGGER), errcode(ERRCODE_AMBIGUOUS_PARAMETER), - errmsg("invalid break point index"), + errmsg("invalid breakpoint index"), errdetail("the given index is either outside the range or already disabled"), errcause("try to disabled a breakpoint that's already disabled"), erraction("use dbe_pldebugger.info_breakpoints() to show all breakpoints"))); @@ -951,7 +957,7 @@ Datum debug_server_turn_off(PG_FUNCTION_ARGS) PlDebugEntry* entry = has_debug_func(funcOid, &found); if (!found) { ereport(WARNING, (errmodule(MOD_PLDEBUGGER), - errmsg("function %d has not be turned on", funcOid))); + errmsg("function %d has not been turned on", funcOid))); } else { if (entry->func && entry->func->debug) { clean_up_debug_server(entry->func->debug, false, false); @@ -1018,12 +1024,12 @@ static void InterfaceCheck(const char* funcname, bool needAttach) int commIdx = u_sess->plsql_cxt.debug_client->comm_idx; CHECK_DEBUG_COMM_VALID(commIdx); /* if current debug index is not myself during debug, clean up my self */ + uint64 clientSessionId = ENABLE_THREAD_POOL ? u_sess->session_id : t_thrd.proc_cxt.MyProcPid; PlDebuggerComm* debug_comm = &g_instance.pldebug_cxt.debug_comm[commIdx]; DebugClientInfo* client = u_sess->plsql_cxt.debug_client; AutoMutexLock debuglock(&debug_comm->mutex); debuglock.lock(); - if ((debug_comm->clientId != u_sess->session_id && debug_comm->clientId != t_thrd.proc_cxt.MyProcPid) || - !debug_comm->isRunning()) { + if (debug_comm->clientId != clientSessionId || !debug_comm->isRunning()) { client->comm_idx = -1; MemoryContextDelete(client->context); u_sess->plsql_cxt.debug_client = NULL; @@ -1088,7 +1094,7 @@ static PlDebugEntry* add_debug_func(Oid key) } else { ReleaseDebugCommIdx(commIdx); ereport(ERROR, (errmodule(MOD_PLDEBUGGER), - errmsg("function %d has already be turned on", key))); + errmsg("function %d has already been turned on", key))); } return entry; } diff --git a/src/include/utils/plpgsql.h b/src/include/utils/plpgsql.h index 42bc71335..e20298e25 100644 --- a/src/include/utils/plpgsql.h +++ b/src/include/utils/plpgsql.h @@ -1576,7 +1576,7 @@ const int DEBUG_SERVER_PRINT_VAR_FRAMENO_EXCEED = 1; if (!g_instance.pldebug_cxt.debug_comm[idx].Used()) \ ereport(ERROR, \ (errmodule(MOD_PLDEBUGGER), errcode(ERRCODE_PLDEBUGGER_ERROR), \ - errmsg("Debug Comm %d has been released or not turn on yet.", idx))); \ + errmsg("Debug Comm %d has been released or not turned on yet.", idx))); \ } while (0) static const char *stringFromCompileStatus(int strindex) diff --git a/src/test/regress/expected/pl_debugger_client.out b/src/test/regress/expected/pl_debugger_client.out index e46163a37..be46c1f3c 100755 --- a/src/test/regress/expected/pl_debugger_client.out +++ b/src/test/regress/expected/pl_debugger_client.out @@ -75,7 +75,7 @@ begin insert into tmp_holder select breakpointno || ':' || lineno || ':' || query || ':' || enable from dbe_pldebugger.info_breakpoints(); end; $$; -WARNING: lineno must be within the range of [1, MaxLineNumber] Please use dbe_pldebugger.info_code for valid breakpoint candidates +WARNING: lineno must be within the range of [1, MaxLineNumber]. Please use dbe_pldebugger.info_code for valid breakpoint candidates CONTEXT: SQL statement "insert into tmp_holder select * from dbe_pldebugger.add_breakpoint(funcoid, 0)" PL/pgSQL function inline_code_block line 6 at SQL statement WARNING: the given line number does not name a valid breakpoint. Please use dbe_pldebugger.info_code for valid breakpoint candidates @@ -87,7 +87,7 @@ PL/pgSQL function inline_code_block line 9 at SQL statement WARNING: the given line number already contains a valid breakpoint. Please se dbe_pldebugger.info_breakpoints for detail. CONTEXT: SQL statement "insert into tmp_holder select * from dbe_pldebugger.add_breakpoint(funcoid, 15)" PL/pgSQL function inline_code_block line 11 at SQL statement -WARNING: lineno must be within the range of [1, MaxLineNumber] Please use dbe_pldebugger.info_code for valid breakpoint candidates +WARNING: lineno must be within the range of [1, MaxLineNumber]. Please use dbe_pldebugger.info_code for valid breakpoint candidates CONTEXT: SQL statement "insert into tmp_holder select * from dbe_pldebugger.add_breakpoint(funcoid, 50)" PL/pgSQL function inline_code_block line 19 at SQL statement select * from tmp_holder; @@ -1475,13 +1475,13 @@ select dbe_pldebugger.info_code(0); (7 rows) insert into tmp_holder select * from dbe_pldebugger.add_breakpoint(0, 0); -- negative -WARNING: lineno must be within the range of [1, MaxLineNumber] Please use dbe_pldebugger.info_code for valid breakpoint candidates +WARNING: lineno must be within the range of [1, MaxLineNumber]. Please use dbe_pldebugger.info_code for valid breakpoint candidates insert into tmp_holder select * from dbe_pldebugger.add_breakpoint(0, 6); -- headerline insert into tmp_holder select * from dbe_pldebugger.add_breakpoint(0, 6); -- already WARNING: the given line number already contains a valid breakpoint. Please se dbe_pldebugger.info_breakpoints for detail. insert into tmp_holder select * from dbe_pldebugger.add_breakpoint(0, 8); -- ok insert into tmp_holder select * from dbe_pldebugger.add_breakpoint(0, 13); -- negative -WARNING: lineno must be within the range of [1, MaxLineNumber] Please use dbe_pldebugger.info_code for valid breakpoint candidates +WARNING: lineno must be within the range of [1, MaxLineNumber]. Please use dbe_pldebugger.info_code for valid breakpoint candidates insert into tmp_holder select * from dbe_pldebugger.disable_breakpoint(1); -- ok insert into tmp_holder select * from dbe_pldebugger.enable_breakpoint(1); -- ok insert into tmp_holder select * from dbe_pldebugger.delete_breakpoint(0); -- ok @@ -1536,10 +1536,31 @@ select * from dbe_pldebugger.print_var('k'); k | int4 | 0 | | f (1 row) -select * from dbe_pldebugger.abort(); - abort -------- - t +select * from dbe_pldebugger.continue(); + funcoid | funcname | lineno | query +---------+-------------------+--------+---------------------- + 0 | "anonymous block" | 0 | [EXECUTION FINISHED] +(1 row) + +select pg_sleep(1); + pg_sleep +---------- + +(1 row) + +select * from dbe_pldebugger.continue(); +ERROR: must attach a execute func before execute dbe_pldebugger.continue +DETAIL: execute func not attached before execute dbe_pldebugger.continue +select dbe_pldebugger.attach(nodename, port) from debug_info; + attach +------------------------------------------------------------------------------------------------------------------ + (0,"""anonymous block""",6," select oid from pg_proc into funcoid where proname='abs' and prosrc='int8abs';") +(1 row) + +select * from dbe_pldebugger.continue(); + funcoid | funcname | lineno | query +---------+-------------------+--------+---------------------- + 0 | "anonymous block" | 0 | [EXECUTION FINISHED] (1 row) select * from tmp_holder; diff --git a/src/test/regress/expected/pl_debugger_server.out b/src/test/regress/expected/pl_debugger_server.out index 22686fbb4..cdc76b0c2 100755 --- a/src/test/regress/expected/pl_debugger_server.out +++ b/src/test/regress/expected/pl_debugger_server.out @@ -161,7 +161,7 @@ ERROR: must attach a execute func before execute dbe_pldebugger.abort DETAIL: execute func not attached before execute dbe_pldebugger.abort -- turn off without turn on select * from dbe_pldebugger.turn_off(1); -WARNING: function 1 has not be turned on +WARNING: function 1 has not been turned on turn_off ---------- f @@ -683,8 +683,21 @@ begin insert into test_anonymous values(k, 'test2'); end; $$; -ERROR: receive abort message -CONTEXT: PL/pgSQL function inline_code_block line 9 at SQL statement +delete from debug_info; +insert into debug_info select * from dbe_pldebugger.turn_on(0); +do $$ +declare + funcoid oid; + k int; +begin + select oid from pg_proc into funcoid where proname='abs' and prosrc='int8abs'; + perform * from pg_proc where oid = funcoid; + k = test_increment(3); + insert into test_anonymous values(k, 'test'); + k = abs(-k); + insert into test_anonymous values(k, 'test2'); +end; +$$; drop schema pl_debugger cascade; NOTICE: drop cascades to 20 other objects DETAIL: drop cascades to table test diff --git a/src/test/regress/sql/pl_debugger_client.sql b/src/test/regress/sql/pl_debugger_client.sql index 3f0034531..50c9d0e68 100644 --- a/src/test/regress/sql/pl_debugger_client.sql +++ b/src/test/regress/sql/pl_debugger_client.sql @@ -478,6 +478,11 @@ select frameno, funcname, lineno, query from dbe_pldebugger.backtrace(); select * from dbe_pldebugger.info_locals(); select funcname, lineno, query from dbe_pldebugger.finish(); select * from dbe_pldebugger.print_var('k'); -select * from dbe_pldebugger.abort(); +select * from dbe_pldebugger.continue(); + +select pg_sleep(1); +select * from dbe_pldebugger.continue(); +select dbe_pldebugger.attach(nodename, port) from debug_info; +select * from dbe_pldebugger.continue(); select * from tmp_holder; \ No newline at end of file diff --git a/src/test/regress/sql/pl_debugger_server.sql b/src/test/regress/sql/pl_debugger_server.sql index 904342c52..9103b0baa 100644 --- a/src/test/regress/sql/pl_debugger_server.sql +++ b/src/test/regress/sql/pl_debugger_server.sql @@ -444,4 +444,21 @@ begin end; $$; +delete from debug_info; +insert into debug_info select * from dbe_pldebugger.turn_on(0); + +do $$ +declare + funcoid oid; + k int; +begin + select oid from pg_proc into funcoid where proname='abs' and prosrc='int8abs'; + perform * from pg_proc where oid = funcoid; + k = test_increment(3); + insert into test_anonymous values(k, 'test'); + k = abs(-k); + insert into test_anonymous values(k, 'test2'); +end; +$$; + drop schema pl_debugger cascade; From 6535af85f719b062a54cf34e1508bf849e3c9d4d Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Thu, 21 Sep 2023 21:07:24 +0800 Subject: [PATCH 274/304] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E5=8F=8C=E9=9B=86?= =?UTF-8?q?=E7=BE=A4=E5=88=87=E6=8D=A2=E6=96=B0=E4=B8=BB=E6=9C=BA=E7=BE=A4?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E5=9B=9E=E6=94=BE=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../storage/access/transam/xlog.cpp | 52 +++++++++---------- .../storage/access/transam/xloginsert.cpp | 5 +- .../storage/smgr/segment/xlog_atomic_op.cpp | 5 +- src/include/access/xlog.h | 1 - 4 files changed, 32 insertions(+), 31 deletions(-) diff --git a/src/gausskernel/storage/access/transam/xlog.cpp b/src/gausskernel/storage/access/transam/xlog.cpp index 30d497a05..4636aaabd 100755 --- a/src/gausskernel/storage/access/transam/xlog.cpp +++ b/src/gausskernel/storage/access/transam/xlog.cpp @@ -649,8 +649,8 @@ static XLogRecPtr XLogInsertRecordGroup(XLogRecData *rdata, XLogRecPtr fpw_lsn) ereport(ERROR, (errcode(ERRCODE_CASE_NOT_FOUND), errmsg("cannot make new WAL entries during recovery"))); } - if (!SSXLogInsertAllowed()) { - ereport(FATAL, (errmsg("SS standby cannot insert XLOG entries"))); + if (SS_REPLICATION_STANDBY_CLUSTER) { + ereport(FATAL, (errmsg("SS dorado standby cluster cannot insert XLOG entries"))); } START_CRIT_SECTION(); @@ -1088,8 +1088,8 @@ static XLogRecPtr XLogInsertRecordSingle(XLogRecData *rdata, XLogRecPtr fpw_lsn) ereport(ERROR, (errcode(ERRCODE_CASE_NOT_FOUND), errmsg("cannot make new WAL entries during recovery"))); } - if (!SSXLogInsertAllowed()) { - ereport(FATAL, (errmsg("SS standby cannot insert XLOG entries"))); + if (SS_REPLICATION_STANDBY_CLUSTER) { + ereport(FATAL, (errmsg("SS dorado standby cluster cannot insert XLOG entries"))); } /* ---------- @@ -3194,7 +3194,7 @@ void XLogWaitFlush(XLogRecPtr recptr) return; } - if (!SSXLogInsertAllowed()) { + if (SS_REPLICATION_STANDBY_CLUSTER) { return; } @@ -3404,7 +3404,7 @@ bool XLogBackgroundFlush(void) return false; } - if (!SSXLogInsertAllowed()) { + if (SS_REPLICATION_STANDBY_CLUSTER) { return false; } @@ -3724,8 +3724,8 @@ static int XLogFileInitInternal(XLogSegNo logsegno, bool *use_existent, bool use } } - if (!SSXLogInsertAllowed()) { - ereport(FATAL, (errmsg("SS standby cannot init xlog files due to DSS"))); + if (SS_REPLICATION_STANDBY_CLUSTER) { + ereport(FATAL, (errmsg("SS dorado standby cluster cannot init xlog files due to DSS"))); } /* @@ -4988,8 +4988,9 @@ static void UpdateLastRemovedPtr(const char *filename) */ static void RemoveOldXlogFiles(XLogSegNo segno, XLogRecPtr endptr) { - if (!SSXLogInsertAllowed()) + if (SS_REPLICATION_STANDBY_CLUSTER) { return; + } DIR *xldir = NULL; struct dirent *xlde = NULL; @@ -5406,6 +5407,13 @@ static XLogRecord *ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, in if (t_thrd.xlog_cxt.StandbyMode && !dummyStandbyMode && !t_thrd.xlog_cxt.recoveryTriggered) { continue; } else { + if (u_sess->attr.attr_storage.HaModuleDebug) { + ereport(LOG, + (errmsg("startup replay xlog ready to exit, standbymode is %s, dummyStandbyMode is %s, recoveryTriggered is %s", + t_thrd.xlog_cxt.StandbyMode ? "true" : "false", + dummyStandbyMode ? "true" : "false", + t_thrd.xlog_cxt.recoveryTriggered ? "true" : "false"))); + } return NULL; } } @@ -10721,6 +10729,10 @@ void StartupXLOG(void) if (ENABLE_DMS && ENABLE_REFORM && !SS_PRIMARY_DEMOTED && !SS_REPLICATION_STANDBY_CLUSTER) { StartupWaitReform(); } + + if (SS_REPLICATION_MAIN_STANBY_NODE) { + SendPostmasterSignal(PMSIGNAL_UPDATE_PROMOTING); + } } void CopyXlogForForceFinishRedo(XLogSegNo logSegNo, uint32 termId, XLogReaderState *xlogreader, @@ -11692,9 +11704,9 @@ void CreateCheckPoint(int flags) (errcode(ERRCODE_INVALID_TRANSACTION_STATE), errmsg("can't create a checkpoint during recovery"))); } - if (!SSXLogInsertAllowed()) { + if (SS_REPLICATION_STANDBY_CLUSTER) { ereport(ERROR, - (errcode(ERRCODE_INVALID_TRANSACTION_STATE), errmsg("can't create a checkpoint on SS standby node"))); + (errcode(ERRCODE_INVALID_TRANSACTION_STATE), errmsg("can't create a checkpoint on SS dorado standby cluster"))); } /* allow standby do checkpoint only after it has promoted AND has finished recovery. */ @@ -17407,7 +17419,9 @@ bool CheckForFailoverTrigger(void) */ ResetSlotLSNEndRecovery(slotname); - SendPostmasterSignal(PMSIGNAL_UPDATE_PROMOTING); + if (!SS_REPLICATION_MAIN_STANBY_NODE) { + SendPostmasterSignal(PMSIGNAL_UPDATE_PROMOTING); + } ret = true; goto exit; } @@ -19164,20 +19178,6 @@ XLogRecPtr GetFlushMainStandby() return flushptr; } -/* SS bans data write unless current node is: - * 1. normal: running as primary; - * 2. switchover: primary demoting, doing shutdown checkpoint; - * 3. switchover: standby promoting, doing StartupXLOG; - * 4. switchover: standby promoted, running as primary de facto, - * waiting for DMS reformer thread to update its role. - * 5. failover: doing StartupXLOG - */ -bool SSXLogInsertAllowed() -{ - /* allow xlog write as long as to different VGs */ - return true; -} - bool SSModifySharedLunAllowed() { if (!ENABLE_DMS || diff --git a/src/gausskernel/storage/access/transam/xloginsert.cpp b/src/gausskernel/storage/access/transam/xloginsert.cpp index efb8335c4..ac16c7b13 100755 --- a/src/gausskernel/storage/access/transam/xloginsert.cpp +++ b/src/gausskernel/storage/access/transam/xloginsert.cpp @@ -35,6 +35,7 @@ #include "utils/guc.h" #include "pg_trace.h" #include "replication/logical.h" +#include "replication/ss_cluster_replication.h" #include "pgstat.h" #include "access/ustore/knl_upage.h" @@ -94,8 +95,8 @@ void XLogBeginInsert(void) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("XLogBeginInsert was already called"))); - if (!SSXLogInsertAllowed()) { - ereport(LOG, (errmsg("SS standby cannot insert XLOG entries"))); + if (SS_REPLICATION_STANDBY_CLUSTER) { + ereport(LOG, (errmsg("SS dorado standby cluster cannot insert XLOG entries"))); return; } diff --git a/src/gausskernel/storage/smgr/segment/xlog_atomic_op.cpp b/src/gausskernel/storage/smgr/segment/xlog_atomic_op.cpp index 45cdee7e5..e3b253725 100644 --- a/src/gausskernel/storage/smgr/segment/xlog_atomic_op.cpp +++ b/src/gausskernel/storage/smgr/segment/xlog_atomic_op.cpp @@ -30,6 +30,7 @@ #include "storage/smgr/segment.h" #include "utils/palloc.h" +#include "replication/ss_cluster_replication.h" /* * In segment-page storage, an atomic operation may cross multiple functions, touching multiple pages. For example, @@ -396,8 +397,8 @@ void XLogAtomicOpStart() errhint("cannot make new WAL entries during recovery"))); } - if (!SSXLogInsertAllowed()) { - ereport(FATAL, (errmsg("SS standby cannot make new WAL entries"))); + if (SS_REPLICATION_STANDBY_CLUSTER) { + ereport(FATAL, (errmsg("SS dorado standby cluster cannot make new WAL entries"))); } XLogAtomicOpMgr->XLogStart(); diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h index be17c9ed6..f90e03ab4 100755 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@ -687,7 +687,6 @@ extern bool RecoveryInProgress(void); extern bool HotStandbyActive(void); extern bool HotStandbyActiveInReplay(void); extern bool XLogInsertAllowed(void); -extern bool SSXLogInsertAllowed(void); extern bool SSModifySharedLunAllowed(void); extern void GetXLogReceiptTime(TimestampTz* rtime, bool* fromStream); extern XLogRecPtr GetXLogReplayRecPtr(TimeLineID* targetTLI, XLogRecPtr* ReplayReadPtr = NULL); From b472eb0cd37721e7855e56986d45fa80545a447b Mon Sep 17 00:00:00 2001 From: bowenliu Date: Tue, 19 Sep 2023 16:41:24 +0800 Subject: [PATCH 275/304] fix reform timeout due to twophasecleaner and backend term deadlock --- src/gausskernel/ddes/adapter/ss_dms_callback.cpp | 10 +++------- src/gausskernel/process/postmaster/twophasecleaner.cpp | 7 +++++++ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/gausskernel/ddes/adapter/ss_dms_callback.cpp b/src/gausskernel/ddes/adapter/ss_dms_callback.cpp index 66dbc94fb..5929d493b 100644 --- a/src/gausskernel/ddes/adapter/ss_dms_callback.cpp +++ b/src/gausskernel/ddes/adapter/ss_dms_callback.cpp @@ -389,9 +389,6 @@ static int CBSwitchoverPromote(void *db_handle, unsigned char origPrimaryId) if (g_instance.dms_cxt.SSClusterState == NODESTATE_STANDBY_PROMOTED) { /* flush control file primary id in advance to save new standby's waiting time */ SSSavePrimaryInstId(SS_MY_INST_ID); - - SSReadControlFile(REFORM_CTRL_PAGE); - Assert(SSGetPrimaryInstId() == SS_MY_INST_ID); ereport(LOG, (errmodule(MOD_DMS), errmsg("[SS switchover] Standby promote: success, set new primary:%d.", SS_MY_INST_ID))); return DMS_SUCCESS; @@ -429,16 +426,15 @@ static void CBSwitchoverResult(void *db_handle, int result) static int SetPrimaryIdOnStandby(int primary_id) { - g_instance.dms_cxt.SSReformerControl.primaryInstId = primary_id; - for (int ntries = 0;; ntries++) { SSReadControlFile(REFORM_CTRL_PAGE); /* need to double check */ if (g_instance.dms_cxt.SSReformerControl.primaryInstId == primary_id) { ereport(LOG, (errmodule(MOD_DMS), - errmsg("[SS %s] Reform success, this is a standby:%d confirming new primary:%d.", - SS_PERFORMING_SWITCHOVER ? "switchover" : "reform", SS_MY_INST_ID, primary_id))); + errmsg("[SS %s] Reform success, this is a standby:%d confirming new primary:%d, confirm ntries=%d.", + SS_PERFORMING_SWITCHOVER ? "switchover" : "reform", SS_MY_INST_ID, primary_id, ntries))); return DMS_SUCCESS; } else { + SSSavePrimaryInstId(primary_id); if (ntries >= WAIT_REFORM_CTRL_REFRESH_TRIES) { ereport(ERROR, (errmodule(MOD_DMS), errmsg("[SS %s] Failed to confirm new primary: %d," diff --git a/src/gausskernel/process/postmaster/twophasecleaner.cpp b/src/gausskernel/process/postmaster/twophasecleaner.cpp index 52698e049..5051e3710 100644 --- a/src/gausskernel/process/postmaster/twophasecleaner.cpp +++ b/src/gausskernel/process/postmaster/twophasecleaner.cpp @@ -205,6 +205,12 @@ NON_EXEC_STATIC void TwoPhaseCleanerMain() if (t_thrd.tpcleaner_cxt.shutdown_requested) { /* Normal exit from the twophasecleaner is here */ + ereport(LOG, (errmsg("TwoPhaseCleaner exits via SIGTERM"))); + proc_exit(0); + } + + if (SS_PRIMARY_DEMOTING) { + ereport(LOG, (errmsg("TwoPhaseCleaner exits via SS global var"))); proc_exit(0); } @@ -326,6 +332,7 @@ static void TwoPCShutdownHandler(SIGNAL_ARGS) { int save_errno = errno; + ereport(LOG, (errmsg("TwoPhaseCleaner received SIGTERM"))); t_thrd.tpcleaner_cxt.shutdown_requested = true; if (t_thrd.proc) From 3a6e031cc101b817c55d206ce122eb4c902efbc4 Mon Sep 17 00:00:00 2001 From: dongning12 Date: Fri, 22 Sep 2023 16:34:04 +0800 Subject: [PATCH 276/304] =?UTF-8?q?=E3=80=90=E8=B5=84=E6=BA=90=E6=B1=A0?= =?UTF-8?q?=E5=8C=96=E3=80=91dms=E6=8E=A8=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/ddes/ddes_commit_id | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gausskernel/ddes/ddes_commit_id b/src/gausskernel/ddes/ddes_commit_id index 6f5522f0e..86996c95c 100644 --- a/src/gausskernel/ddes/ddes_commit_id +++ b/src/gausskernel/ddes/ddes_commit_id @@ -1,3 +1,3 @@ -dms_commit_id=1f1cacdc3285a8b43b1e73f0f439f79dda7630d0 +dms_commit_id=a6c2fef3b8c08c11e4cee1adcacafabdf3b2f56b dss_commit_id=a4d52e326a5291ca1ef8d2d22993a561f13f45a8 cbb_commit_id=a7f55f6371988695578304bfbd3450f6f31c6630 \ No newline at end of file From 17b3f58f933584de717ea18421e57c13d192016d Mon Sep 17 00:00:00 2001 From: xuxinxin Date: Fri, 22 Sep 2023 18:04:42 +0800 Subject: [PATCH 277/304] =?UTF-8?q?=E6=8E=A8=E8=BF=9BDSScommit=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/ddes/ddes_commit_id | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gausskernel/ddes/ddes_commit_id b/src/gausskernel/ddes/ddes_commit_id index 86996c95c..fb48bf880 100644 --- a/src/gausskernel/ddes/ddes_commit_id +++ b/src/gausskernel/ddes/ddes_commit_id @@ -1,3 +1,3 @@ dms_commit_id=a6c2fef3b8c08c11e4cee1adcacafabdf3b2f56b -dss_commit_id=a4d52e326a5291ca1ef8d2d22993a561f13f45a8 +dss_commit_id=13bb5ba33820961455093f838977c54ba0867f28 cbb_commit_id=a7f55f6371988695578304bfbd3450f6f31c6630 \ No newline at end of file From b705b9bff3af25e59caa19066a7b56101ff60e6e Mon Sep 17 00:00:00 2001 From: totaj Date: Fri, 22 Sep 2023 19:14:11 +0800 Subject: [PATCH 278/304] Fix execute job bug. --- src/common/backend/catalog/pg_job.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/common/backend/catalog/pg_job.cpp b/src/common/backend/catalog/pg_job.cpp index 256e436c8..5d31b57cd 100644 --- a/src/common/backend/catalog/pg_job.cpp +++ b/src/common/backend/catalog/pg_job.cpp @@ -1066,6 +1066,7 @@ void execute_job(int4 job_id) FlushErrorState(); t_thrd.utils_cxt.CurrentResourceOwner = save; + ResourceOwnerRelease(save, RESOURCE_RELEASE_BEFORE_LOCKS, false, true); /* Update last_end_date and job_status='f' and failure_count++ */ update_pg_job_info(job_id, Pgjob_Fail, start_date, new_next_date, edata->message, is_scheduler_job); elog_job_detail(job_id, what, Pgjob_Fail, edata->message); From 5c60de8ba4d8f57f6bbafdc4048f00a340ff49dc Mon Sep 17 00:00:00 2001 From: chendong76 <1209756284@qq.com> Date: Thu, 21 Sep 2023 11:25:26 +0800 Subject: [PATCH 279/304] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E6=8C=89=E9=9C=80?= =?UTF-8?q?=E5=9B=9E=E6=94=BE=E4=B8=AD=EF=BC=8C=E6=9C=AA=E5=8D=87=E4=B8=BB?= =?UTF-8?q?=E5=A4=87=E6=9C=BAlsn=E6=A0=A1=E9=AA=8C=E5=A4=B1=E8=B4=A5?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98;=E5=88=A0=E9=99=A4=E6=8C=89?= =?UTF-8?q?=E9=9C=80=E5=9B=9E=E6=94=BE=E6=94=AF=E6=8C=81=E5=8D=87=E7=BA=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/utils/init/globals.cpp | 1 - src/common/backend/utils/init/miscinit.cpp | 11 -- src/common/backend/utils/misc/guc-file.l | 1 - .../backend/utils/misc/guc/guc_storage.cpp | 14 +- .../ddes/adapter/ss_dms_bufmgr.cpp | 21 +-- .../ddes/adapter/ss_dms_callback.cpp | 11 +- .../ddes/adapter/ss_dms_recovery.cpp | 4 +- .../ddes/adapter/ss_reform_common.cpp | 132 ++---------------- .../ddes/adapter/ss_transaction.cpp | 29 ++++ .../process/postmaster/postmaster.cpp | 4 +- .../process/threadpool/knl_instance.cpp | 1 + .../ondemand_extreme_rto/dispatcher.cpp | 13 +- .../storage/access/transam/xlog.cpp | 26 +++- src/gausskernel/storage/buffer/bufmgr.cpp | 17 ++- src/include/ddes/dms/ss_common_attr.h | 10 +- src/include/ddes/dms/ss_dms_recovery.h | 1 + src/include/ddes/dms/ss_reform_common.h | 2 +- src/include/ddes/dms/ss_transaction.h | 2 + src/include/miscadmin.h | 2 - 19 files changed, 113 insertions(+), 189 deletions(-) diff --git a/src/common/backend/utils/init/globals.cpp b/src/common/backend/utils/init/globals.cpp index 45c289ada..7a607f672 100644 --- a/src/common/backend/utils/init/globals.cpp +++ b/src/common/backend/utils/init/globals.cpp @@ -89,7 +89,6 @@ const uint32 TIMESCALE_DB_VERSION_NUM = 92904; const uint32 MULTI_CHARSET_VERSION_NUM = 92903; const uint32 NBTREE_INSERT_OPTIMIZATION_VERSION_NUM = 92902; const uint32 NBTREE_DEDUPLICATION_VERSION_NUM = 92902; -const uint32 ONDEMAND_REDO_VERSION_NUM = 92901; const uint32 SRF_FUSION_VERSION_NUM = 92847; const uint32 INDEX_HINT_VERSION_NUM = 92845; const uint32 INNER_UNIQUE_VERSION_NUM = 92845; diff --git a/src/common/backend/utils/init/miscinit.cpp b/src/common/backend/utils/init/miscinit.cpp index 329706f92..bd2e1763a 100644 --- a/src/common/backend/utils/init/miscinit.cpp +++ b/src/common/backend/utils/init/miscinit.cpp @@ -2040,17 +2040,6 @@ void register_backend_version(uint32 backend_version){ } } -void SSUpgradeFileBeforeCommit() -{ - // upgrade reform control file - if (pg_atomic_read_u32(&WorkingGrandVersionNum) < ONDEMAND_REDO_VERSION_NUM) { - if (SS_PRIMARY_MODE) { - SSReadControlFile(REFORM_CTRL_PAGE); - SSSaveReformerCtrl(true); - } - } -} - /* * Check whether the version contains the backend_version parameter. */ diff --git a/src/common/backend/utils/misc/guc-file.l b/src/common/backend/utils/misc/guc-file.l index fc79beb87..3a7096b9c 100644 --- a/src/common/backend/utils/misc/guc-file.l +++ b/src/common/backend/utils/misc/guc-file.l @@ -330,7 +330,6 @@ ProcessConfigFile(GucContext context) case MASTER_THREAD: { if (strcmp(item->name, "upgrade_mode") == 0) { if (strcmp(pre_value, "0") != 0 && strcmp(post_value, "0") == 0) { - SSUpgradeFileBeforeCommit(); pg_atomic_write_u32(&WorkingGrandVersionNum, GRAND_VERSION_NUM); } } diff --git a/src/common/backend/utils/misc/guc/guc_storage.cpp b/src/common/backend/utils/misc/guc/guc_storage.cpp index 1ee3c7c7f..bc44a8843 100755 --- a/src/common/backend/utils/misc/guc/guc_storage.cpp +++ b/src/common/backend/utils/misc/guc/guc_storage.cpp @@ -215,7 +215,6 @@ static bool check_ss_rdma_work_config(char** newval, void** extra, GucSource sou static bool check_ss_dss_vg_name(char** newval, void** extra, GucSource source); static bool check_ss_dss_conn_path(char** newval, void** extra, GucSource source); static bool check_ss_enable_ssl(bool* newval, void** extra, GucSource source); -static bool check_ss_enable_ondemand_recovery(bool* newval, void** extra, GucSource source); static bool check_normal_cluster_replication_config_para(char** newval, void** extra, GucSource source); static bool check_ss_cluster_replication_control_para(bool* newval, void** extra, GucSource source); @@ -1054,7 +1053,7 @@ static void InitStorageConfigureNamesBool() GUC_SUPERUSER_ONLY}, &g_instance.attr.attr_storage.dms_attr.enable_ondemand_recovery, false, - check_ss_enable_ondemand_recovery, + NULL, NULL, NULL}, @@ -6320,17 +6319,6 @@ static bool check_ss_enable_ssl(bool *newval, void **extra, GucSource source) return true; } -static bool check_ss_enable_ondemand_recovery(bool* newval, void** extra, GucSource source) -{ - if (*newval) { - if (pg_atomic_read_u32(&WorkingGrandVersionNum) < ONDEMAND_REDO_VERSION_NUM) { - ereport(ERROR, (errmsg("Do not allow enable ondemand_recovery if openGauss run in old version."))); - return false; - } - } - return true; -} - #ifdef USE_ASSERT_CHECKING static void assign_ss_enable_verify_page(bool newval, void *extra) { diff --git a/src/gausskernel/ddes/adapter/ss_dms_bufmgr.cpp b/src/gausskernel/ddes/adapter/ss_dms_bufmgr.cpp index 816195ea1..d9df56c34 100644 --- a/src/gausskernel/ddes/adapter/ss_dms_bufmgr.cpp +++ b/src/gausskernel/ddes/adapter/ss_dms_bufmgr.cpp @@ -484,16 +484,15 @@ Buffer DmsReadPage(Buffer buffer, LWLockMode mode, ReadBufferMode read_mode, boo } // standby node must notify primary node for prepare lastest page in ondemand recovery - if (SS_STANDBY_ONDEMAND_RECOVERY) { - while (!SSOndemandRequestPrimaryRedo(buf_desc->tag)) { - SSReadControlFile(REFORM_CTRL_PAGE); - if (SS_STANDBY_ONDEMAND_NORMAL) { - break; // ondemand recovery finish, skip - } else if (SS_STANDBY_ONDEMAND_BUILD) { - return 0; // in new reform - } - // still need requset page + while (SS_STANDBY_ONDEMAND_NOT_NORMAL) { + /* in new reform */ + if (unlikely(SS_STANDBY_ONDEMAND_BUILD)) { + return 0; + } + if (SSOndemandRequestPrimaryRedo(buf_desc->tag)) { + break; } + SSReadControlFile(REFORM_CTRL_PAGE); } if (!StartReadPage(buf_desc, mode)) { @@ -507,7 +506,9 @@ bool SSOndemandRequestPrimaryRedo(BufferTag tag) dms_context_t dms_ctx; int32 redo_status = ONDEMAND_REDO_TIMEOUT; - if (!SS_STANDBY_ONDEMAND_RECOVERY) { + if (unlikely(SS_STANDBY_ONDEMAND_BUILD)) { + return false; + } else if (SS_STANDBY_ONDEMAND_NORMAL) { return true; } diff --git a/src/gausskernel/ddes/adapter/ss_dms_callback.cpp b/src/gausskernel/ddes/adapter/ss_dms_callback.cpp index d371ec6ad..0c23b9214 100644 --- a/src/gausskernel/ddes/adapter/ss_dms_callback.cpp +++ b/src/gausskernel/ddes/adapter/ss_dms_callback.cpp @@ -430,8 +430,6 @@ static void CBSwitchoverResult(void *db_handle, int result) static int SetPrimaryIdOnStandby(int primary_id) { - g_instance.dms_cxt.SSReformerControl.primaryInstId = primary_id; - for (int ntries = 0;; ntries++) { SSReadControlFile(REFORM_CTRL_PAGE); /* need to double check */ if (g_instance.dms_cxt.SSReformerControl.primaryInstId == primary_id) { @@ -462,6 +460,7 @@ static int CBSaveStableList(void *db_handle, unsigned long long list_stable, uns unsigned long long list_in, unsigned int save_ctrl) { int primary_id = (int)reformer_id; + LWLockAcquire(ControlFileLock, LW_EXCLUSIVE); g_instance.dms_cxt.SSReformerControl.primaryInstId = primary_id; g_instance.dms_cxt.SSReformerControl.list_stable = list_stable; int ret = DMS_ERROR; @@ -473,7 +472,8 @@ static int CBSaveStableList(void *db_handle, unsigned long long list_stable, uns Assert(g_instance.dms_cxt.SSClusterState == NODESTATE_STANDBY_PROMOTED || g_instance.dms_cxt.SSClusterState == NODESTATE_STANDBY_FAILOVER_PROMOTING); } - SSSaveReformerCtrl(); + SSUpdateReformerCtrl(); + LWLockRelease(ControlFileLock); Assert(g_instance.dms_cxt.SSReformerControl.primaryInstId == (int)primary_id); ereport(LOG, (errmodule(MOD_DMS), errmsg("[SS %s] set current instance:%d as primary.", SS_PERFORMING_SWITCHOVER ? "switchover" : "reform", primary_id))); @@ -483,6 +483,7 @@ static int CBSaveStableList(void *db_handle, unsigned long long list_stable, uns } ret = DMS_SUCCESS; } else { /* we are on standby */ + LWLockRelease(ControlFileLock); ret = SetPrimaryIdOnStandby(primary_id); } return ret; @@ -1116,6 +1117,9 @@ static int32 CBProcessBroadcast(void *db_handle, dms_broadcast_context_t *broad_ case BCAST_SEND_SNAPSHOT: ret = SSUpdateLatestSnapshotOfStandby(data, len); break; + case BCAST_RELOAD_REFORM_CTRL_PAGE: + ret = SSReloadReformCtrlPage(len); + break; default: ereport(WARNING, (errmodule(MOD_DMS), errmsg("invalid broadcast operate type"))); ret = DMS_ERROR; @@ -1863,7 +1867,6 @@ static int CBReformDoneNotify(void *db_handle) g_instance.dms_cxt.SSRecoveryInfo.startup_reform = false; g_instance.dms_cxt.SSRecoveryInfo.restart_failover_flag = false; g_instance.dms_cxt.SSRecoveryInfo.failover_ckpt_status = NOT_ACTIVE; - SSReadControlFile(REFORM_CTRL_PAGE); Assert(g_instance.dms_cxt.SSRecoveryInfo.in_flushcopy == false); g_instance.dms_cxt.SSReformInfo.new_bitmap = g_instance.dms_cxt.SSReformerControl.list_stable; ereport(LOG, (errmsg("[SS reform] new cluster node bitmap: %lld", g_instance.dms_cxt.SSReformInfo.new_bitmap))); diff --git a/src/gausskernel/ddes/adapter/ss_dms_recovery.cpp b/src/gausskernel/ddes/adapter/ss_dms_recovery.cpp index f64bc60db..d965542ce 100644 --- a/src/gausskernel/ddes/adapter/ss_dms_recovery.cpp +++ b/src/gausskernel/ddes/adapter/ss_dms_recovery.cpp @@ -51,8 +51,10 @@ int SSGetPrimaryInstId() void SSSavePrimaryInstId(int id) { + LWLockAcquire(ControlFileLock, LW_EXCLUSIVE); g_instance.dms_cxt.SSReformerControl.primaryInstId = id; - SSSaveReformerCtrl(); + SSUpdateReformerCtrl(); + LWLockRelease(ControlFileLock); } void SSWaitStartupExit() diff --git a/src/gausskernel/ddes/adapter/ss_reform_common.cpp b/src/gausskernel/ddes/adapter/ss_reform_common.cpp index 38d411a29..1871fceaf 100644 --- a/src/gausskernel/ddes/adapter/ss_reform_common.cpp +++ b/src/gausskernel/ddes/adapter/ss_reform_common.cpp @@ -249,120 +249,13 @@ void SSDoradoGetInstidList() closedir(dssdir); } -static void SSSaveOldReformerCtrl() -{ - ss_reformer_ctrl_t new_ctrl = g_instance.dms_cxt.SSReformerControl; - ss_old_reformer_ctrl_t old_ctrl = {new_ctrl.list_stable, new_ctrl.primaryInstId, new_ctrl.crc}; - - int len = sizeof(ss_old_reformer_ctrl_t); - int write_size = (int)BUFFERALIGN(len); - char buffer[write_size] __attribute__((__aligned__(ALIGNOF_BUFFER))) = { 0 }; - char *fname[2]; - int fd = -1; - - errno_t err = memcpy_s(&buffer, write_size, &old_ctrl, len); - securec_check(err, "\0", "\0"); - - INIT_CRC32C(((ss_old_reformer_ctrl_t *)buffer)->crc); - COMP_CRC32C(((ss_old_reformer_ctrl_t *)buffer)->crc, (char *)buffer, offsetof(ss_old_reformer_ctrl_t, crc)); - FIN_CRC32C(((ss_old_reformer_ctrl_t *)buffer)->crc); - - fname[0] = XLOG_CONTROL_FILE_BAK; - fname[1] = XLOG_CONTROL_FILE; - - for (int i = 0; i < BAK_CTRL_FILE_NUM; i++) { - if (i == 0) { - fd = BasicOpenFile(fname[i], O_CREAT | O_RDWR | PG_BINARY, S_IRUSR | S_IWUSR); - } else { - fd = BasicOpenFile(fname[i], O_RDWR | PG_BINARY, S_IRUSR | S_IWUSR); - } - - if (fd < 0) { - ereport(FATAL, (errcode_for_file_access(), errmsg("could not open control file \"%s\": %m", fname[i]))); - } - - SSWriteInstanceControlFile(fd, buffer, REFORM_CTRL_PAGE, write_size); - if (close(fd)) { - ereport(PANIC, (errcode_for_file_access(), errmsg("could not close control file: %m"))); - } - } -} - -static bool SSReadOldReformerCtrl() -{ - ss_reformer_ctrl_t *new_ctrl = &g_instance.dms_cxt.SSReformerControl; - ss_old_reformer_ctrl_t old_ctrl; - pg_crc32c crc; - int fd = -1; - bool retry = false; - char *fname = XLOG_CONTROL_FILE; - -loop: - fd = BasicOpenFile(fname, O_RDWR | PG_BINARY, S_IRUSR | S_IWUSR); - if (fd < 0) { - ereport(FATAL, (errcode_for_file_access(), errmsg("could not open control file \"%s\": %m", fname))); - } - - off_t seekpos = (off_t)BLCKSZ * REFORM_CTRL_PAGE; - int len = sizeof(ss_old_reformer_ctrl_t); - - int read_size = (int)BUFFERALIGN(len); - char buffer[read_size] __attribute__((__aligned__(ALIGNOF_BUFFER))); - if (pread(fd, buffer, read_size, seekpos) != read_size) { - ereport(PANIC, (errcode_for_file_access(), errmsg("could not read from control file: %m"))); - } - - errno_t rc = memcpy_s(&old_ctrl, len, buffer, len); - securec_check(rc, "", ""); - if (close(fd) < 0) { - ereport(PANIC, (errcode_for_file_access(), errmsg("could not close control file: %m"))); - } - - /* Now check the CRC. */ - INIT_CRC32C(crc); - COMP_CRC32C(crc, (char *)&old_ctrl, offsetof(ss_old_reformer_ctrl_t, crc)); - FIN_CRC32C(crc); - - if (!EQ_CRC32C(crc, old_ctrl.crc)) { - if (retry == false) { - ereport(WARNING, - (errmsg("control file \"%s\" contains incorrect checksum in upgrade mode, try backup file", fname))); - fname = XLOG_CONTROL_FILE_BAK; - retry = true; - goto loop; - } else { - ereport(WARNING, - (errmsg("backup control file \"%s\" contains incorrect checksum in upgrade mode, " - "try again in post-upgrade mode", fname))); - return false; - } - } - - // new params set to initial value - new_ctrl->version = REFORM_CTRL_VERSION; - new_ctrl->recoveryInstId = INVALID_INSTANCEID; - new_ctrl->clusterStatus = CLUSTER_NORMAL; - - // exist param inherit - new_ctrl->primaryInstId = old_ctrl.primaryInstId; - new_ctrl->list_stable = old_ctrl.list_stable; - new_ctrl->crc = old_ctrl.crc; - - return true; -} - -void SSSaveReformerCtrl(bool force) +void SSUpdateReformerCtrl() { int fd = -1; int len; errno_t err = EOK; char *fname[2]; - if ((pg_atomic_read_u32(&WorkingGrandVersionNum) < ONDEMAND_REDO_VERSION_NUM) && !force) { - SSSaveOldReformerCtrl(); - return; - } - len = sizeof(ss_reformer_ctrl_t); int write_size = (int)BUFFERALIGN(len); char buffer[write_size] __attribute__((__aligned__(ALIGNOF_BUFFER))) = { 0 }; @@ -405,24 +298,12 @@ void SSReadControlFile(int id, bool updateDmsCtx) int read_size = 0; int len = 0; fname = XLOG_CONTROL_FILE; - - if ((pg_atomic_read_u32(&WorkingGrandVersionNum) < ONDEMAND_REDO_VERSION_NUM) && (id == REFORM_CTRL_PAGE)) { - if (SSReadOldReformerCtrl()) { - return; - } - - // maybe primary node already upgrade pg_control file, sleep and try read in lastest mode again - if (SS_STANDBY_MODE) { - pg_usleep(5000000); /* 5 sec */ - goto loop; - } else { - ereport(PANIC, (errmsg("incorrect checksum in control file"))); - } - } + LWLockAcquire(ControlFileLock, LW_EXCLUSIVE); loop: fd = BasicOpenFile(fname, O_RDWR | PG_BINARY, S_IRUSR | S_IWUSR); if (fd < 0) { + LWLockRelease(ControlFileLock); ereport(FATAL, (errcode_for_file_access(), errmsg("could not open control file \"%s\": %m", fname))); } @@ -437,6 +318,7 @@ void SSReadControlFile(int id, bool updateDmsCtx) read_size = (int)BUFFERALIGN(len); char buffer[read_size] __attribute__((__aligned__(ALIGNOF_BUFFER))); if (pread(fd, buffer, read_size, seekpos) != read_size) { + LWLockRelease(ControlFileLock); ereport(PANIC, (errcode_for_file_access(), errmsg("could not read from control file: %m"))); } @@ -444,6 +326,7 @@ void SSReadControlFile(int id, bool updateDmsCtx) rc = memcpy_s(&g_instance.dms_cxt.SSReformerControl, len, buffer, len); securec_check(rc, "", ""); if (close(fd) < 0) { + LWLockRelease(ControlFileLock); ereport(PANIC, (errcode_for_file_access(), errmsg("could not close control file: %m"))); } @@ -459,9 +342,11 @@ void SSReadControlFile(int id, bool updateDmsCtx) retry = true; goto loop; } else { + LWLockRelease(ControlFileLock); ereport(FATAL, (errmsg("incorrect checksum in control file"))); } } + g_instance.dms_cxt.SSRecoveryInfo.cluster_ondemand_status= g_instance.dms_cxt.SSReformerControl.clusterStatus; } else { ControlFileData* controlFile = NULL; ControlFileData tempControlFile; @@ -474,6 +359,7 @@ void SSReadControlFile(int id, bool updateDmsCtx) rc = memcpy_s(controlFile, (size_t)len, buffer, (size_t)len); securec_check(rc, "", ""); if (close(fd) < 0) { + LWLockRelease(ControlFileLock); ereport(PANIC, (errcode_for_file_access(), errmsg("could not close control file: %m"))); } @@ -489,6 +375,7 @@ void SSReadControlFile(int id, bool updateDmsCtx) retry = true; goto loop; } else { + LWLockRelease(ControlFileLock); ereport(FATAL, (errmsg("incorrect checksum in control file"))); } } @@ -497,6 +384,7 @@ void SSReadControlFile(int id, bool updateDmsCtx) g_instance.dms_cxt.ckptRedo = controlFile->checkPointCopy.redo; } } + LWLockRelease(ControlFileLock); } void SSClearSegCache() diff --git a/src/gausskernel/ddes/adapter/ss_transaction.cpp b/src/gausskernel/ddes/adapter/ss_transaction.cpp index 642ebbe0b..247cfee09 100644 --- a/src/gausskernel/ddes/adapter/ss_transaction.cpp +++ b/src/gausskernel/ddes/adapter/ss_transaction.cpp @@ -28,6 +28,7 @@ #include "storage/buf/bufmgr.h" #include "storage/smgr/segment_internal.h" #include "ddes/dms/ss_transaction.h" +#include "ddes/dms/ss_reform_common.h" #include "ddes/dms/ss_dms_bufmgr.h" #include "storage/sinvaladt.h" #include "replication/libpqsw.h" @@ -502,6 +503,16 @@ void SSIsPageHitDms(RelFileNode& node, BlockNumber page, int pagesNum, uint64 *p ereport(DEBUG1, (errmsg("SS get page map success, buffer_id = %u.", page))); } +int SSReloadReformCtrlPage(uint32 len) +{ + if (unlikely(len != sizeof(SSBroadcastCmdOnly))) { + return DMS_ERROR; + } + + SSReadControlFile(REFORM_CTRL_PAGE); + return DMS_SUCCESS; +} + int SSCheckDbBackends(char *data, uint32 len, char *output_msg, uint32 *output_msg_len) { if (unlikely(len != sizeof(SSBroadcastDbBackends))) { @@ -553,6 +564,24 @@ bool SSCheckDbBackendsFromAllStandby(Oid dbid) return false; } +void SSRequestAllStandbyReloadReformCtrlPage() +{ + dms_context_t dms_ctx; + InitDmsContext(&dms_ctx); + int ret; + SSBroadcastCmdOnly ssmsg; + ssmsg.type = BCAST_RELOAD_REFORM_CTRL_PAGE; + do { + ret = dms_broadcast_msg(&dms_ctx, (char *)&ssmsg, sizeof(SSBroadcastCmdOnly), + (unsigned char)false, SS_BROADCAST_WAIT_ONE_SECOND); + + if (ret == DMS_SUCCESS) { + return; + } + pg_usleep(5000L); + } while (ret != DMS_SUCCESS); +} + void SSSendSharedInvalidMessages(const SharedInvalidationMessage *msgs, int n) { dms_context_t dms_ctx; diff --git a/src/gausskernel/process/postmaster/postmaster.cpp b/src/gausskernel/process/postmaster/postmaster.cpp index 2a4e68afb..535cee05a 100644 --- a/src/gausskernel/process/postmaster/postmaster.cpp +++ b/src/gausskernel/process/postmaster/postmaster.cpp @@ -10029,8 +10029,10 @@ static void sigusr1_handler(SIGNAL_ARGS) ereport(LOG, (errmsg("Failover between two dorado cluster start, change current run mode to primary_cluster"))); g_instance.attr.attr_common.cluster_run_mode = RUN_MODE_PRIMARY; + LWLockAcquire(ControlFileLock, LW_EXCLUSIVE); g_instance.dms_cxt.SSReformerControl.clusterRunMode = RUN_MODE_PRIMARY; - SSSaveReformerCtrl(); + SSUpdateReformerCtrl(); + LWLockRelease(ControlFileLock); t_thrd.xlog_cxt.server_mode = PRIMARY_MODE; SetHaShmemData(); } diff --git a/src/gausskernel/process/threadpool/knl_instance.cpp b/src/gausskernel/process/threadpool/knl_instance.cpp index 518e71f7f..dd85b2659 100755 --- a/src/gausskernel/process/threadpool/knl_instance.cpp +++ b/src/gausskernel/process/threadpool/knl_instance.cpp @@ -184,6 +184,7 @@ static void knl_g_dms_init(knl_g_dms_context *dms_cxt) dms_cxt->SSReformInfo.redo_total_bytes = 0; dms_cxt->SSClusterState = NODESTATE_NORMAL; dms_cxt->SSRecoveryInfo.recovery_inst_id = INVALID_INSTANCEID; + dms_cxt->SSRecoveryInfo.cluster_ondemand_status= CLUSTER_NORMAL; dms_cxt->SSRecoveryInfo.recovery_pause_flag = true; dms_cxt->SSRecoveryInfo.failover_ckpt_status = NOT_ACTIVE; dms_cxt->SSRecoveryInfo.new_primary_reset_walbuf_flag = false; diff --git a/src/gausskernel/storage/access/transam/ondemand_extreme_rto/dispatcher.cpp b/src/gausskernel/storage/access/transam/ondemand_extreme_rto/dispatcher.cpp index 9e8e0336c..a19a0a374 100644 --- a/src/gausskernel/storage/access/transam/ondemand_extreme_rto/dispatcher.cpp +++ b/src/gausskernel/storage/access/transam/ondemand_extreme_rto/dispatcher.cpp @@ -78,6 +78,7 @@ #include "utils/atomic.h" #include "pgstat.h" #include "ddes/dms/ss_reform_common.h" +#include "ddes/dms/ss_transaction.h" #ifdef PGXC #include "pgxc/pgxc.h" @@ -1908,14 +1909,18 @@ void WaitRedoFinish() pmState = PM_RUN; write_stderr_with_prefix("[On-demand] LOG: database system is ready to accept connections"); + g_instance.dms_cxt.SSRecoveryInfo.cluster_ondemand_status= CLUSTER_IN_ONDEMAND_REDO; + /* for other nodes in cluster */ + LWLockAcquire(ControlFileLock, LW_EXCLUSIVE); + g_instance.dms_cxt.SSReformerControl.clusterStatus = CLUSTER_IN_ONDEMAND_REDO; + SSUpdateReformerCtrl(); + LWLockRelease(ControlFileLock); + SSRequestAllStandbyReloadReformCtrlPage(); + SpinLockAcquire(&t_thrd.shemem_ptr_cxt.XLogCtl->info_lck); t_thrd.shemem_ptr_cxt.XLogCtl->IsOnDemandBuildDone = true; SpinLockRelease(&t_thrd.shemem_ptr_cxt.XLogCtl->info_lck); - /* for other nodes in cluster */ - g_instance.dms_cxt.SSReformerControl.clusterStatus = CLUSTER_IN_ONDEMAND_REDO; - SSSaveReformerCtrl(); - #ifdef USE_ASSERT_CHECKING XLogRecPtr minStart = MAX_XLOG_REC_PTR; XLogRecPtr minEnd = MAX_XLOG_REC_PTR; diff --git a/src/gausskernel/storage/access/transam/xlog.cpp b/src/gausskernel/storage/access/transam/xlog.cpp index 30d497a05..9f586fa90 100755 --- a/src/gausskernel/storage/access/transam/xlog.cpp +++ b/src/gausskernel/storage/access/transam/xlog.cpp @@ -155,6 +155,7 @@ #include "vectorsonic/vsonichash.h" #include "ddes/dms/ss_reform_common.h" +#include "ddes/dms/ss_transaction.h" #include "ddes/dms/ss_dms_recovery.h" #include "ddes/dms/ss_dms_bufmgr.h" #include "storage/file/fio_device.h" @@ -9630,10 +9631,14 @@ void StartupXLOG(void) t_thrd.xlog_cxt.InRecovery == true) { if (SSOndemandRecoveryExitNormal) { g_instance.dms_cxt.SSRecoveryInfo.in_ondemand_recovery = true; + g_instance.dms_cxt.SSRecoveryInfo.cluster_ondemand_status= CLUSTER_IN_ONDEMAND_BUILD; /* for other nodes in cluster and ondeamnd recovery failed */ + LWLockAcquire(ControlFileLock, LW_EXCLUSIVE); g_instance.dms_cxt.SSReformerControl.clusterStatus = CLUSTER_IN_ONDEMAND_BUILD; g_instance.dms_cxt.SSReformerControl.recoveryInstId = g_instance.dms_cxt.SSRecoveryInfo.recovery_inst_id; - SSSaveReformerCtrl(); + SSUpdateReformerCtrl(); + LWLockRelease(ControlFileLock); + SSRequestAllStandbyReloadReformCtrlPage(); SetOndemandExtremeRtoMode(); ereport(LOG, (errmsg("[On-demand] replayed in extreme rto ondemand recovery mode"))); } else { @@ -9643,8 +9648,11 @@ void StartupXLOG(void) } if (SS_PRIMARY_MODE || SS_REPLICATION_MAIN_STANBY_NODE) { - g_instance.dms_cxt.SSReformerControl.clusterRunMode = (ClusterRunMode)g_instance.attr.attr_common.cluster_run_mode; - SSSaveReformerCtrl(); + LWLockAcquire(ControlFileLock, LW_EXCLUSIVE); + g_instance.dms_cxt.SSReformerControl.clusterRunMode = + (ClusterRunMode)g_instance.attr.attr_common.cluster_run_mode; + SSUpdateReformerCtrl(); + LWLockRelease(ControlFileLock); } ReadRemainSegsFile(); @@ -10206,8 +10214,12 @@ void StartupXLOG(void) ereport(LOG, (errmsg("redo is not required"))); if (SS_IN_ONDEMAND_RECOVERY) { g_instance.dms_cxt.SSRecoveryInfo.in_ondemand_recovery = false; + g_instance.dms_cxt.SSRecoveryInfo.cluster_ondemand_status= CLUSTER_NORMAL; + LWLockAcquire(ControlFileLock, LW_EXCLUSIVE); g_instance.dms_cxt.SSReformerControl.clusterStatus = CLUSTER_NORMAL; - SSSaveReformerCtrl(); + SSUpdateReformerCtrl(); + LWLockRelease(ControlFileLock); + SSRequestAllStandbyReloadReformCtrlPage(); } } } @@ -10695,9 +10707,13 @@ void StartupXLOG(void) } if (SS_PRIMARY_MODE) { + g_instance.dms_cxt.SSRecoveryInfo.cluster_ondemand_status= CLUSTER_NORMAL; /* for other nodes in cluster */ + LWLockAcquire(ControlFileLock, LW_EXCLUSIVE); g_instance.dms_cxt.SSReformerControl.clusterStatus = CLUSTER_NORMAL; - SSSaveReformerCtrl(); + SSUpdateReformerCtrl(); + LWLockRelease(ControlFileLock); + SSRequestAllStandbyReloadReformCtrlPage(); } ereport(LOG, (errmsg("redo done, nextXid: " XID_FMT ", startupMaxXid: " XID_FMT ", recentLocalXmin: " XID_FMT diff --git a/src/gausskernel/storage/buffer/bufmgr.cpp b/src/gausskernel/storage/buffer/bufmgr.cpp index 65601d55a..65ba79f4d 100644 --- a/src/gausskernel/storage/buffer/bufmgr.cpp +++ b/src/gausskernel/storage/buffer/bufmgr.cpp @@ -2423,16 +2423,15 @@ Buffer ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber fork /* DMS: Try get page remote */ if (ENABLE_DMS) { // standby node must notify primary node for prepare lastest page in ondemand recovery - if (SS_STANDBY_ONDEMAND_RECOVERY) { - while (!SSOndemandRequestPrimaryRedo(bufHdr->tag)) { - SSReadControlFile(REFORM_CTRL_PAGE); - if (SS_STANDBY_ONDEMAND_NORMAL) { - break; // ondemand recovery finish, skip - } else if (SS_STANDBY_ONDEMAND_BUILD) { - return 0; // in new reform - } - // still need requset page + while (SS_STANDBY_ONDEMAND_NOT_NORMAL) { + /* in new reform */ + if (unlikely(SS_STANDBY_ONDEMAND_BUILD)) { + return 0; + } + if (SSOndemandRequestPrimaryRedo(bufHdr->tag)) { + break; } + SSReadControlFile(REFORM_CTRL_PAGE); } MarkReadHint(bufHdr->buf_id, relpersistence, isExtend, pblk); if (mode != RBM_FOR_REMOTE && relpersistence != RELPERSISTENCE_TEMP && !isLocalBuf) { diff --git a/src/include/ddes/dms/ss_common_attr.h b/src/include/ddes/dms/ss_common_attr.h index 924794838..eb2588efe 100644 --- a/src/include/ddes/dms/ss_common_attr.h +++ b/src/include/ddes/dms/ss_common_attr.h @@ -99,16 +99,17 @@ /* Mode in dorado hyperreplication and dms enabled as follow */ #define SS_CLUSTER_ONDEMAND_NOT_NORAML \ - (ENABLE_DMS && (g_instance.dms_cxt.SSReformerControl.clusterStatus != CLUSTER_NORMAL)) + (ENABLE_DMS && (g_instance.dms_cxt.SSRecoveryInfo.cluster_ondemand_status!= CLUSTER_NORMAL)) #define SS_CLUSTER_ONDEMAND_BUILD \ - (ENABLE_DMS && (g_instance.dms_cxt.SSReformerControl.clusterStatus == CLUSTER_IN_ONDEMAND_BUILD)) + (ENABLE_DMS && (g_instance.dms_cxt.SSRecoveryInfo.cluster_ondemand_status== CLUSTER_IN_ONDEMAND_BUILD)) #define SS_CLUSTER_ONDEMAND_RECOVERY \ - (ENABLE_DMS && (g_instance.dms_cxt.SSReformerControl.clusterStatus == CLUSTER_IN_ONDEMAND_REDO)) + (ENABLE_DMS && (g_instance.dms_cxt.SSRecoveryInfo.cluster_ondemand_status== CLUSTER_IN_ONDEMAND_REDO)) #define SS_CLUSTER_ONDEMAND_NORMAL \ - (ENABLE_DMS && (g_instance.dms_cxt.SSReformerControl.clusterStatus == CLUSTER_NORMAL)) + (ENABLE_DMS && (g_instance.dms_cxt.SSRecoveryInfo.cluster_ondemand_status== CLUSTER_NORMAL)) #define SS_STANDBY_ONDEMAND_BUILD (SS_STANDBY_MODE && SS_CLUSTER_ONDEMAND_BUILD) #define SS_STANDBY_ONDEMAND_RECOVERY (SS_STANDBY_MODE && SS_CLUSTER_ONDEMAND_RECOVERY) #define SS_STANDBY_ONDEMAND_NORMAL (SS_STANDBY_MODE && SS_CLUSTER_ONDEMAND_NORMAL) +#define SS_STANDBY_ONDEMAND_NOT_NORMAL (SS_STANDBY_MODE && SS_CLUSTER_ONDEMAND_NOT_NORAML) /* DMS_BUF_NEED_LOAD */ #define BUF_NEED_LOAD 0x1 @@ -164,6 +165,7 @@ typedef enum SSBroadcastOp { BCAST_DDLLOCKRELEASE_ALL, BCAST_CHECK_DB_BACKENDS, BCAST_SEND_SNAPSHOT, + BCAST_RELOAD_REFORM_CTRL_PAGE, BCAST_END } SSBroadcastOp; diff --git a/src/include/ddes/dms/ss_dms_recovery.h b/src/include/ddes/dms/ss_dms_recovery.h index ba4caf5e5..7f2a1321d 100644 --- a/src/include/ddes/dms/ss_dms_recovery.h +++ b/src/include/ddes/dms/ss_dms_recovery.h @@ -88,6 +88,7 @@ typedef struct ss_recovery_info { volatile failover_ckpt_status_t failover_ckpt_status; char recovery_xlog_dir[MAXPGPATH]; int recovery_inst_id; + volatile SSGlobalClusterState cluster_ondemand_status; int instid_list[DMS_MAX_INSTANCE]; LWLock* update_seg_lock; bool new_primary_reset_walbuf_flag; diff --git a/src/include/ddes/dms/ss_reform_common.h b/src/include/ddes/dms/ss_reform_common.h index a11429d1d..ec5173648 100644 --- a/src/include/ddes/dms/ss_reform_common.h +++ b/src/include/ddes/dms/ss_reform_common.h @@ -44,7 +44,7 @@ int SSReadXlogInternal(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, XL XLogReaderState *SSXLogReaderAllocate(XLogPageReadCB pagereadfunc, void *private_data, Size alignedSize); void SSGetRecoveryXlogPath(); void SSDoradoGetInstidList(); -void SSSaveReformerCtrl(bool force = false); +void SSUpdateReformerCtrl(); void SSReadControlFile(int id, bool updateDmsCtx = false); void SSClearSegCache(); int SSCancelTransactionOfAllStandby(SSBroadcastOp type); diff --git a/src/include/ddes/dms/ss_transaction.h b/src/include/ddes/dms/ss_transaction.h index 723da9f92..dabe4fecf 100644 --- a/src/include/ddes/dms/ss_transaction.h +++ b/src/include/ddes/dms/ss_transaction.h @@ -122,5 +122,7 @@ bool SSCheckDbBackendsFromAllStandby(Oid dbid); void SSStandbyUpdateRedirectInfo(); void SSSendLatestSnapshotToStandby(TransactionId xmin, TransactionId xmax, CommitSeqNo csn); int SSUpdateLatestSnapshotOfStandby(char *data, uint32 len); +int SSReloadReformCtrlPage(uint32 len); +void SSRequestAllStandbyReloadReformCtrlPage(); #endif diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h index abe48258f..dd1de5b57 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h @@ -42,7 +42,6 @@ extern const uint32 PARAM_MARK_VERSION_NUM; extern const uint32 TIMESCALE_DB_VERSION_NUM; extern const uint32 NBTREE_INSERT_OPTIMIZATION_VERSION_NUM; extern const uint32 NBTREE_DEDUPLICATION_VERSION_NUM; -extern const uint32 ONDEMAND_REDO_VERSION_NUM; extern const uint32 MULTI_CHARSET_VERSION_NUM; extern const uint32 SRF_FUSION_VERSION_NUM; extern const uint32 INNER_UNIQUE_VERSION_NUM; @@ -139,7 +138,6 @@ extern const uint32 GB18030_2022_VERSION_NUM; extern void register_backend_version(uint32 backend_version); extern bool contain_backend_version(uint32 version_number); -extern void SSUpgradeFileBeforeCommit(); #define INPLACE_UPGRADE_PRECOMMIT_VERSION 1 From 33c9b4eb06b217c28b86de0f731ada205894bfd5 Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Sat, 23 Sep 2023 11:24:53 +0800 Subject: [PATCH 280/304] =?UTF-8?q?ss=20probackup=E9=80=82=E9=85=8Ddouble?= =?UTF-8?q?=20write=20initdb=E7=9A=84=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/pg_probackup/dir.cpp | 13 +++++++++---- src/bin/pg_probackup/pg_probackupc.h | 1 + src/bin/pg_probackup/util.cpp | 20 ++++++++++++++++---- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/src/bin/pg_probackup/dir.cpp b/src/bin/pg_probackup/dir.cpp index fb4a766b8..95d500f7b 100644 --- a/src/bin/pg_probackup/dir.cpp +++ b/src/bin/pg_probackup/dir.cpp @@ -1312,10 +1312,15 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba if (dir->external_dir_num != 0) continue; - if (is_dss_type(dir->type) && is_ss_xlog(dir->rel_path)) { - ss_createdir(dir->rel_path, instance_config.dss.vgdata, - instance_config.dss.vglog); - continue; + if (is_dss_type(dir->type)) { + if (is_ss_xlog(dir->rel_path)) { + ss_createdir(dir->rel_path, instance_config.dss.vgdata, instance_config.dss.vglog); + continue; + } + + if (ss_create_if_doublewrite(dir, instance_config.dss.vgdata, instance_config.dss.instance_id)) { + continue; + } } /* tablespace_map exists */ diff --git a/src/bin/pg_probackup/pg_probackupc.h b/src/bin/pg_probackup/pg_probackupc.h index 9358aa6e7..13df3f742 100644 --- a/src/bin/pg_probackup/pg_probackupc.h +++ b/src/bin/pg_probackup/pg_probackupc.h @@ -390,6 +390,7 @@ extern void get_redo(const char *pgdata_path, RedoParams *redo); extern void parse_vgname_args(const char* args); extern bool is_ss_xlog(const char *ss_dir); extern void ss_createdir(const char *ss_dir, const char *vgdata, const char *vglog); +extern bool ss_create_if_doublewrite(pgFile* dir, const char* vgdata, int instance_id); extern char* xstrdup(const char* s); extern void set_min_recovery_point(pgFile *file, const char *fullpath, XLogRecPtr stop_backup_lsn); diff --git a/src/bin/pg_probackup/util.cpp b/src/bin/pg_probackup/util.cpp index aefb8503e..c6bb47ddb 100644 --- a/src/bin/pg_probackup/util.cpp +++ b/src/bin/pg_probackup/util.cpp @@ -560,9 +560,6 @@ is_ss_xlog(const char *ss_dir) rc = sprintf_s(ss_xlog, sizeof(ss_xlog), "%s%d", "pg_xlog", instance_id); securec_check_ss_c(rc, "\0", "\0"); - rc = sprintf_s(ss_doublewrite, sizeof(ss_doublewrite), "%s%d", "pg_doublewrite", instance_id); - securec_check_ss_c(rc, "\0", "\0"); - rc = sprintf_s(ss_notify, sizeof(ss_notify), "%s%d", "pg_notify", instance_id); securec_check_ss_c(rc, "\0", "\0"); @@ -571,7 +568,6 @@ is_ss_xlog(const char *ss_dir) if (IsDssMode() && strlen(instance_config.dss.vglog) && (pg_strcasecmp(ss_dir, ss_xlog) == 0 || - pg_strcasecmp(ss_dir, ss_doublewrite) == 0 || pg_strcasecmp(ss_dir, ss_notify) == 0 || pg_strcasecmp(ss_dir, ss_notify) == 0)) { return true; @@ -598,6 +594,22 @@ ss_createdir(const char *ss_dir, const char *vgdata, const char *vglog) } } +bool +ss_create_if_doublewrite(pgFile* dir, const char* vgdata, int instance_id) +{ + char ss_doublewrite[MAXPGPATH];; + errno_t rc = sprintf_s(ss_doublewrite, sizeof(ss_doublewrite), "%s%d", "pg_doublewrite", instance_id); + securec_check_ss_c(rc, "\0", "\0"); + if (pg_strcasecmp(dir->rel_path, ss_doublewrite) == 0) { + rc = sprintf_s(ss_doublewrite, sizeof(ss_doublewrite), "%s/%s", vgdata, dir->rel_path); + securec_check_ss_c(rc, "\0", "\0"); + dir_create_dir(ss_doublewrite, DIR_PERMISSION); + return true; + } else { + return false; + } +} + /* * Rewrite minRecoveryPoint of pg_control in backup directory. minRecoveryPoint * 'as-is' is not to be trusted. From 458f24a717d2203a3aef8c12bc74e94228b3db56 Mon Sep 17 00:00:00 2001 From: totaj Date: Sat, 23 Sep 2023 14:54:18 +0800 Subject: [PATCH 281/304] Fix multi update bug. --- .../optimizer/rewrite/rewriteHandler.cpp | 2 +- src/test/regress/expected/multi_update.out | 9 +++++++++ src/test/regress/sql/multi_update.sql | 13 ++++++++++++- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/gausskernel/optimizer/rewrite/rewriteHandler.cpp b/src/gausskernel/optimizer/rewrite/rewriteHandler.cpp index 38073a378..dd5b07062 100644 --- a/src/gausskernel/optimizer/rewrite/rewriteHandler.cpp +++ b/src/gausskernel/optimizer/rewrite/rewriteHandler.cpp @@ -3819,7 +3819,7 @@ static List* RewriteQuery(Query* parsetree, List* rewrite_events) bool rewriteView = false; List* rewriteRelations = NIL; - result_relation = linitial_int(parsetree->resultRelations); + result_relation = linitial2_int(parsetree->resultRelations); if (result_relation == 0) return rewritten; diff --git a/src/test/regress/expected/multi_update.out b/src/test/regress/expected/multi_update.out index 965c0dd0b..8095bce3b 100644 --- a/src/test/regress/expected/multi_update.out +++ b/src/test/regress/expected/multi_update.out @@ -1452,5 +1452,14 @@ drop function trigger_func_update_multiview1(); drop view multiview1; drop view multiview2; drop view multiview3; +-- multi update with join +CREATE TABLE t1 (f1 int); +CREATE TABLE t2 (f1 int); +INSERT INTO t2 VALUES (1); +CREATE VIEW v1 AS SELECT * FROM t2; +UPDATE t2 AS A NATURAL JOIN v1 B SET B.f1 = 1; +drop table t1; +drop view v1; +drop table t2; \c regression drop database multiupdate; diff --git a/src/test/regress/sql/multi_update.sql b/src/test/regress/sql/multi_update.sql index 779663810..6b2eb215a 100644 --- a/src/test/regress/sql/multi_update.sql +++ b/src/test/regress/sql/multi_update.sql @@ -639,5 +639,16 @@ drop function trigger_func_update_multiview1(); drop view multiview1; drop view multiview2; drop view multiview3; +-- multi update with join +CREATE TABLE t1 (f1 int); +CREATE TABLE t2 (f1 int); +INSERT INTO t2 VALUES (1); +CREATE VIEW v1 AS SELECT * FROM t2; + +UPDATE t2 AS A NATURAL JOIN v1 B SET B.f1 = 1; +drop table t1; +drop view v1; +drop table t2; + \c regression -drop database multiupdate; \ No newline at end of file +drop database multiupdate; From 7f14ee168761eabf7c01430b7658215e16c82a33 Mon Sep 17 00:00:00 2001 From: wuyuechuan Date: Sat, 23 Sep 2023 16:51:05 +0800 Subject: [PATCH 282/304] dont generate recovery.conf when rt(time/lsn/xid) is not bigger than backup(time/lsn/xid --- src/bin/pg_probackup/restore.cpp | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/bin/pg_probackup/restore.cpp b/src/bin/pg_probackup/restore.cpp index 1593c1133..25a139be6 100644 --- a/src/bin/pg_probackup/restore.cpp +++ b/src/bin/pg_probackup/restore.cpp @@ -1430,10 +1430,17 @@ create_recovery_conf(time_t backup_id, * in "the past" relatively to latest state in the archive? * We will get a replica that is "in the future" to the master. * We accept this risk because its probability is low. + * + * if rt recovery is not bigger than backup, we dont need to + * generate recovery.conf file + * + * rt is malloc 0 before */ - pitr_requested = !backup->stream || rt->time_string || - rt->xid_string || rt->lsn_string || rt->target_name || - target_immediate || target_latest || restore_command_provided; + pitr_requested = + !backup->stream || backup->recovery_time < rt->target_time || backup->recovery_xid < rt->target_xid || + backup->stop_lsn < rt->target_lsn || rt->target_name || target_immediate || target_latest || + restore_command_provided; + /* No need to generate recovery.conf at all. */ if (!pitr_requested) From 2b759e1132c00e81c7860539d0d054cd6509484a Mon Sep 17 00:00:00 2001 From: totaj Date: Sat, 23 Sep 2023 16:24:21 +0800 Subject: [PATCH 283/304] Fix prepare with user defined var. --- src/gausskernel/optimizer/rewrite/rewriteHandler.cpp | 6 ++++++ src/test/regress/expected/mysql_compatibility.out | 10 ++++++++-- src/test/regress/parallel_schedule0 | 2 +- src/test/regress/parallel_schedule0C | 2 +- src/test/regress/sql/mysql_compatibility.sql | 6 ++++++ 5 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/gausskernel/optimizer/rewrite/rewriteHandler.cpp b/src/gausskernel/optimizer/rewrite/rewriteHandler.cpp index 38073a378..957b745d8 100644 --- a/src/gausskernel/optimizer/rewrite/rewriteHandler.cpp +++ b/src/gausskernel/optimizer/rewrite/rewriteHandler.cpp @@ -5166,10 +5166,16 @@ List* QueryRewritePrepareStmt(Query* parsetree) (errcode(ERRCODE_UNRECOGNIZED_NODE_TYPE), errmsg("userdefined variable in prepare statement must be text type."))); } + if (value->constvalue == (Datum)0) { + ereport(ERROR, (errcode(ERRCODE_UNRECOGNIZED_NODE_TYPE), errmsg("Query was empty"))); + } sqlstr = TextDatumGetCString(value->constvalue); raw_parsetree_list = pg_parse_query(sqlstr); + if (raw_parsetree_list == NIL) { + ereport(ERROR, (errcode(ERRCODE_UNRECOGNIZED_NODE_TYPE), errmsg("Query was empty"))); + } if (raw_parsetree_list->length != 1) { ereport(ERROR, diff --git a/src/test/regress/expected/mysql_compatibility.out b/src/test/regress/expected/mysql_compatibility.out index 0652ba0ef..efa5baeab 100644 --- a/src/test/regress/expected/mysql_compatibility.out +++ b/src/test/regress/expected/mysql_compatibility.out @@ -1,6 +1,6 @@ -- B db compatibility case drop database if exists B_db; -NOTICE: database "B_db" does not exist, skipping +NOTICE: database "b_db" does not exist, skipping create database B_db dbcompatibility 'B'; --------------------concat-------------------- -- concat case in A db compatibility @@ -634,7 +634,7 @@ select timestampdiff(second, '2018-01-01', now()); (1 row) select timestampdiff(microsecond, '2018-01-01', now()); - timestamp_diff +--?.* --?.* --?.* (1 row) @@ -797,6 +797,12 @@ select @v1, @1a_b.2$3, @a_b.2$3, @_ab.2$3, @.ab_2$3, @$ab.2_3; drop table if exists test1; NOTICE: table "test1" does not exist, skipping create table test1 (f1 int,f2 int,f3 text); +-- prepare with empty var or not exist var +set @v_empty := ''; +prepare stmt_empty as @v_empty; +ERROR: Query was empty +prepare stmt_empty as @does_not_exist; +ERROR: Query was empty -- insertStmt set @v2 := 'insert into test1 values(1, 2, 123)'; prepare stmt2 as @v2; diff --git a/src/test/regress/parallel_schedule0 b/src/test/regress/parallel_schedule0 index 6935231d7..bdb4208fc 100644 --- a/src/test/regress/parallel_schedule0 +++ b/src/test/regress/parallel_schedule0 @@ -1032,7 +1032,7 @@ test: fdw_audit test: gs_global_config_audit test: detail declare_multiple_variable # mysql_function_prepare must precede mysql_function/mysql_function_2, and mysql_function_clearup must follow mysql_function/mysql_function_2 -test: gs_dump_encrypt substr mysql_function_prepare +test: gs_dump_encrypt substr mysql_function_prepare mysql_compatibility test: composite_datum_record mysql_function mysql_function_2 b_comments mysql_syntax mysql_condition mysql_delimiter mysql_delimiter_fix mysql_indexhint mysql_trigger mysql_signal mysql_resignal test: join_test_alias alter_ctable_compress diff --git a/src/test/regress/parallel_schedule0C b/src/test/regress/parallel_schedule0C index a7ef5ba08..f7ae2e75f 100644 --- a/src/test/regress/parallel_schedule0C +++ b/src/test/regress/parallel_schedule0C @@ -119,7 +119,7 @@ test: fdw_audit test: gs_global_config_audit test: detail declare_multiple_variable # mysql_function_prepare must precede mysql_function/mysql_function_2, and mysql_function_clearup must follow mysql_function/mysql_function_2 -test: gs_dump_encrypt substr mysql_function_prepare +test: gs_dump_encrypt substr mysql_function_prepare mysql_compatibility test: composite_datum_record mysql_function mysql_function_2 b_comments mysql_syntax mysql_condition mysql_delimiter mysql_delimiter_fix mysql_indexhint mysql_trigger test: join_test_alias alter_ctable_compress diff --git a/src/test/regress/sql/mysql_compatibility.sql b/src/test/regress/sql/mysql_compatibility.sql index 97cbd6aca..cb193d5ab 100644 --- a/src/test/regress/sql/mysql_compatibility.sql +++ b/src/test/regress/sql/mysql_compatibility.sql @@ -216,6 +216,12 @@ select @v1, @1a_b.2$3, @a_b.2$3, @_ab.2$3, @.ab_2$3, @$ab.2_3; drop table if exists test1; create table test1 (f1 int,f2 int,f3 text); + +-- prepare with empty var or not exist var +set @v_empty := ''; +prepare stmt_empty as @v_empty; +prepare stmt_empty as @does_not_exist; + -- insertStmt set @v2 := 'insert into test1 values(1, 2, 123)'; prepare stmt2 as @v2; From dc62692049ba94b90a7f7bc177b2ce5d09277dbe Mon Sep 17 00:00:00 2001 From: he-shaoyu Date: Mon, 25 Sep 2023 09:37:22 +0800 Subject: [PATCH 284/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8DrepallocFileInfo?= =?UTF-8?q?=E5=87=BD=E6=95=B0=E5=86=85=E5=AD=98=E6=B3=84=E6=BC=8F=E9=A3=8E?= =?UTF-8?q?=E9=99=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/gsqlerr/scanEreport.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/bin/gsqlerr/scanEreport.cpp b/src/bin/gsqlerr/scanEreport.cpp index 3946e64ad..20b6cf6db 100644 --- a/src/bin/gsqlerr/scanEreport.cpp +++ b/src/bin/gsqlerr/scanEreport.cpp @@ -595,10 +595,12 @@ typedef struct { static FileInfo *repallocFileInfo(FileInfo *file_infos, const int new_num, const int old_num) { if (new_num < 0 || (size_t)new_num >= ((0x3fffffff) / sizeof(FileInfo *))) { + free(file_infos); return NULL; } FileInfo *tmps = (FileInfo *)malloc(sizeof(FileInfo) * new_num); if (tmps == NULL) { + free(file_infos); return NULL; } for (int i = 0; i < old_num; i++) { From b3b833b510706eda4c387dbe7a076ce84f5ab304 Mon Sep 17 00:00:00 2001 From: April01xxx Date: Fri, 22 Sep 2023 19:44:25 +0800 Subject: [PATCH 285/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E8=A1=A8=E4=B8=8A?= =?UTF-8?q?=E6=9C=89BRU=E8=A7=A6=E5=8F=91=E5=99=A8=E6=97=B6=E7=9A=84?= =?UTF-8?q?=E4=B8=80=E4=BA=9B=E9=97=AE=E9=A2=98=E3=80=82=20=20=20=20=20BRU?= =?UTF-8?q?=E8=A7=A6=E5=8F=91=E5=99=A8=E4=B8=AD=E5=8F=AF=E8=83=BD=E6=9B=B4?= =?UTF-8?q?=E6=96=B0=E5=88=86=E5=8C=BA=E9=94=AE=EF=BC=8C=E6=AD=A4=E6=97=B6?= =?UTF-8?q?=E6=88=91=E4=BB=AC=E9=9C=80=E8=A6=81=E9=87=8D=E6=96=B0=E8=AE=A1?= =?UTF-8?q?=E7=AE=97=E5=85=83=E7=BB=84=E6=89=80=E5=B1=9E=E7=9A=84=E5=88=86?= =?UTF-8?q?=E5=8C=BA=EF=BC=9B=20=20=20=20=20=E5=BD=93UPDATE=E6=97=B6?= =?UTF-8?q?=E9=81=87=E5=88=B0CONCURRENTLY=20UPDATE/DELETE=E5=9C=BA?= =?UTF-8?q?=E6=99=AF=E6=97=B6=EF=BC=8CMERGE=20INTO=E9=9C=80=E8=A6=81=20=20?= =?UTF-8?q?=20=20=20=E9=87=8D=E6=96=B0=E5=88=A4=E6=96=AD=E6=98=AF=E5=90=A6?= =?UTF-8?q?=E5=8C=B9=E9=85=8D=EF=BC=8C=E8=A7=A6=E5=8F=91=E5=99=A8=E4=B8=AD?= =?UTF-8?q?=E6=89=A7=E8=A1=8CEPQ=E5=90=8E=E6=8A=95=E5=BD=B1=E7=9A=84?= =?UTF-8?q?=E7=BB=93=E6=9E=9C=E9=94=99=E8=AF=AF=EF=BC=8C=E5=8F=AF=E8=83=BD?= =?UTF-8?q?=E5=AF=BC=E8=87=B4=E5=AE=95=E6=9C=BA=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/utils/misc/guc/guc_sql.cpp | 2 +- .../optimizer/commands/trigger.cpp | 24 +- .../runtime/executor/execMerge.cpp | 181 +++++++++++- .../runtime/executor/nodeModifyTable.cpp | 161 +++++------ src/include/commands/trigger.h | 5 +- src/include/executor/node/nodeModifyTable.h | 2 +- src/include/miscadmin.h | 4 +- .../regress/expected/merge_into_deleted.out | 53 ++++ .../merge_into_partition_row_movement.out | 267 ++++++++++++++++++ .../expected/merge_into_selfmodified.out | 133 +++++++++ .../regress/expected/merge_into_updated.out | 140 +++++++++ src/test/regress/parallel_schedule0 | 2 +- src/test/regress/parallel_schedule0A | 2 +- src/test/regress/sql/merge_into_deleted.sql | 44 +++ .../sql/merge_into_partition_row_movement.sql | 182 ++++++++++++ .../regress/sql/merge_into_selfmodified.sql | 91 ++++++ src/test/regress/sql/merge_into_updated.sql | 118 ++++++++ 17 files changed, 1304 insertions(+), 107 deletions(-) create mode 100644 src/test/regress/expected/merge_into_deleted.out create mode 100644 src/test/regress/expected/merge_into_partition_row_movement.out create mode 100644 src/test/regress/expected/merge_into_selfmodified.out create mode 100644 src/test/regress/expected/merge_into_updated.out create mode 100644 src/test/regress/sql/merge_into_deleted.sql create mode 100644 src/test/regress/sql/merge_into_partition_row_movement.sql create mode 100644 src/test/regress/sql/merge_into_selfmodified.sql create mode 100644 src/test/regress/sql/merge_into_updated.sql diff --git a/src/common/backend/utils/misc/guc/guc_sql.cpp b/src/common/backend/utils/misc/guc/guc_sql.cpp index 26115af26..4a557c89f 100755 --- a/src/common/backend/utils/misc/guc/guc_sql.cpp +++ b/src/common/backend/utils/misc/guc/guc_sql.cpp @@ -366,7 +366,7 @@ static const struct behavior_compat_entry behavior_compat_options[OPT_MAX] = { {"unbind_divide_bound", OPT_UNBIND_DIVIDE_BOUND}, {"correct_to_number", OPT_CORRECT_TO_NUMBER}, {"compat_concat_variadic", OPT_CONCAT_VARIADIC}, - {"merge_update_multi", OPT_MEGRE_UPDATE_MULTI}, + {"merge_update_multi", OPT_MERGE_UPDATE_MULTI}, {"convert_string_digit_to_numeric", OPT_CONVERT_TO_NUMERIC}, {"plstmt_implicit_savepoint", OPT_PLSTMT_IMPLICIT_SAVEPOINT}, {"hide_tailing_zero", OPT_HIDE_TAILING_ZERO}, diff --git a/src/gausskernel/optimizer/commands/trigger.cpp b/src/gausskernel/optimizer/commands/trigger.cpp index 7d945c741..f4931891f 100644 --- a/src/gausskernel/optimizer/commands/trigger.cpp +++ b/src/gausskernel/optimizer/commands/trigger.cpp @@ -88,8 +88,6 @@ /* Local function prototypes */ static void ConvertTriggerToFK(CreateTrigStmt* stmt, Oid funcoid); static void SetTriggerFlags(TriggerDesc* trigdesc, const Trigger* trigger); -HeapTuple GetTupleForTrigger(EState* estate, EPQState* epqstate, ResultRelInfo* relinfo, Oid targetPartitionOid, - int2 bucketid, ItemPointer tid, LockTupleMode lockmode, TupleTableSlot** newSlot); static void ReleaseFakeRelation(Relation relation, Partition part, Relation* fakeRelation); static bool TriggerEnabled(EState* estate, ResultRelInfo* relinfo, Trigger* trigger, TriggerEvent event, const Bitmapset* modifiedCols, HeapTuple oldtup, HeapTuple newtup); @@ -2718,7 +2716,7 @@ TupleTableSlot* ExecBRUpdateTriggers(EState* estate, EPQState* epqstate, ResultR #ifdef PGXC HeapTupleHeader datanode_tuphead, #endif - ItemPointer tupleid, TupleTableSlot* slot) + ItemPointer tupleid, TupleTableSlot* slot, TM_Result* result, TM_FailureData* tmfd) { TriggerDesc* trigdesc = get_and_check_trigdesc_value(relinfo->ri_TrigDesc); HeapTuple slottuple = ExecMaterializeSlot(slot); @@ -2780,7 +2778,7 @@ TupleTableSlot* ExecBRUpdateTriggers(EState* estate, EPQState* epqstate, ResultR #endif /* get a copy of the on-disk tuple we are planning to update */ trigtuple = GetTupleForTrigger(estate, epqstate, relinfo, oldPartitionOid, - bucketid, tupleid, lockmode, &newSlot); + bucketid, tupleid, lockmode, &newSlot, result, tmfd); if (trigtuple == NULL) return NULL; /* cancel the update action */ @@ -3031,7 +3029,7 @@ void ExecASTruncateTriggers(EState* estate, ResultRelInfo* relinfo) } HeapTuple GetTupleForTrigger(EState* estate, EPQState* epqstate, ResultRelInfo* relinfo, Oid targetPartitionOid, - int2 bucketid, ItemPointer tid, LockTupleMode lockmode, TupleTableSlot** newSlot) + int2 bucketid, ItemPointer tid, LockTupleMode lockmode, TupleTableSlot** newSlot, TM_Result* tmresultp, TM_FailureData* tmfdp) { Relation relation = relinfo->ri_RelationDesc; HeapTupleData tuple; @@ -3203,6 +3201,13 @@ HeapTuple GetTupleForTrigger(EState* estate, EPQState* epqstate, ResultRelInfo* &tmfd, true, // fake params below are for uheap implementation false, false, NULL, NULL, false); + + /* Let the caller know about the status of this operation */ + if (tmresultp) + *tmresultp = test; + if (tmfdp) + *tmfdp = tmfd; + switch (test) { case TM_SelfUpdated: case TM_SelfModified: @@ -3236,6 +3241,15 @@ HeapTuple GetTupleForTrigger(EState* estate, EPQState* epqstate, ResultRelInfo* (errcode(ERRCODE_T_R_SERIALIZATION_FAILURE), errmsg("could not serialize access due to concurrent update"))); Assert(!ItemPointerEquals(&tmfd.ctid, &tuple.t_self)); + + /* + * Recheck the tuple using EPQ. For MERGE, we leave this + * to the caller (it must do additional rechecking, and + * might end up executing a different action entirely). + */ + if (tmresultp && estate->es_plannedstmt->commandType == CMD_MERGE) + return NULL; + /* it was updated, so look at the updated version */ TupleTableSlot* epqslot = NULL; diff --git a/src/gausskernel/runtime/executor/execMerge.cpp b/src/gausskernel/runtime/executor/execMerge.cpp index 795ed95c6..fa6c71914 100644 --- a/src/gausskernel/runtime/executor/execMerge.cpp +++ b/src/gausskernel/runtime/executor/execMerge.cpp @@ -34,10 +34,43 @@ #include "utils/memutils.h" #include "utils/rel.h" #include "access/heapam.h" +#include "utils/relcache.h" +#include "utils/snapshot.h" static void ExecMergeNotMatched(ModifyTableState* mtstate, EState* estate, TupleTableSlot* slot, char* partExprKeyStr = NULL); static bool ExecMergeMatched(ModifyTableState* mtstate, EState* estate, TupleTableSlot* slot, JunkFilter* junkfilter, ItemPointer tupleid, HeapTupleHeader oldtuple, Oid oldPartitionOid, int2 bucketid, char* partExprKeyStr = NULL); +static LockTupleMode ExecUpdateLockMode(EState *estate, ResultRelInfo *relinfo); + +#define GET_ALL_UPDATED_COLUMNS(relinfo, estate) \ + (bms_union(exec_rt_fetch((relinfo)->ri_RangeTableIndex, estate)->updatedCols, \ + exec_rt_fetch((relinfo)->ri_RangeTableIndex, estate)->extraUpdatedCols)) + +/* + * ExecUpdateLockMode -- find the appropriate UPDATE tuple lock mode for a + * given ResultRelInfo + */ +static LockTupleMode +ExecUpdateLockMode(EState *estate, ResultRelInfo *relinfo) +{ + Bitmapset *keyCols; + Bitmapset *updatedCols; + + /* + * Compute lock mode to use. If columns that are part of the key have not + * been modified, then we can use a weaker lock, allowing for better + * concurrency. + */ + updatedCols = GET_ALL_UPDATED_COLUMNS(relinfo, estate); + keyCols = RelationGetIndexAttrBitmap(relinfo->ri_RelationDesc, + INDEX_ATTR_BITMAP_KEY); + + if (bms_overlap(keyCols, updatedCols)) + return LockTupleExclusive; + + return LockTupleNoKeyExclusive; +} + /* * Perform MERGE. */ @@ -391,10 +424,12 @@ static bool ExecMergeMatched(ModifyTableState* mtstate, EState* estate, TupleTab if (mergeMatchedActionStates != NIL) { MergeActionState* action = (MergeActionState*)linitial(mergeMatchedActionStates); +lmerge_matched: slot = ExecMergeProjQual(mtstate, mergeMatchedActionStates, econtext, slot, slot, estate); if (slot != NULL) { - TM_Result out_result; + TM_Result result = TM_Ok; + TM_FailureData tmfd; (void)ExecUpdate(tupleid, oldPartitionOid, bucketid, @@ -405,11 +440,145 @@ static bool ExecMergeMatched(ModifyTableState* mtstate, EState* estate, TupleTab mtstate, mtstate->canSetTag, partKeyUpdated, - &out_result, - partExprKeyStr); - /* the matched row has been delted or after updated, the row does not matched, change to insert. */ - if (out_result == TM_Deleted || out_result == TM_Updated) { - return false; + &result, + partExprKeyStr, + &tmfd); + + /* + * The matched tuple has been updated or deleted by trigger or + * other session, we have to check the updated version of the + * tuple to see if we want to process it under RC rules. + */ + switch (result) { + case TM_Ok: + /* all good; perform final actions */ + break; + case TM_SelfUpdated: + case TM_SelfModified: + /* The SQL standard disallows this for MERGE, but... */ + if (TransactionIdIsCurrentTransactionId(tmfd.xmax)) { + if (!MERGE_UPDATE_MULTI) + ereport(ERROR, + (errcode(ERRCODE_CARDINALITY_VIOLATION), + errmsg("MERGE command cannot affect row a second time"), + errhint("Ensure that not more than one source row matches any one target row."))); + /* already updated or deleted. */ + break; + } + /* This shouldn't happen */ + elog(ERROR, "attempted to update or delete invisible tuple"); + break; + case TM_Deleted: + if (IsolationUsesXactSnapshot()) + ereport(ERROR, + (errcode(ERRCODE_T_R_SERIALIZATION_FAILURE), + errmsg("could not serialize access due to concurrent delete"))); + + if (resultRelInfo->ri_RelationDesc->rd_rel->relrowmovement) { + Assert(RELATION_IS_PARTITIONED(resultRelInfo->ri_RelationDesc)); + /* + * the may be a row movement update action which delete tuple from original + * partition and insert tuple to new partition or we can add lock on the tuple to + * be delete or updated to avoid throw exception + */ + ereport(ERROR, + (errcode(ERRCODE_TRANSACTION_ROLLBACK), + errmsg("partition table update conflict"), + errdetail("disable row movement of table can avoid this conflict"))); + } + /* + * If the tuple was already deleted, return to let caller + * handle it under NOT MATCHED clauses. + */ + return false; + case TM_Updated: + { + Relation resultRelationDesc; + Relation partRelationDesc; + TupleTableSlot *epqslot; + LockTupleMode lockmode; + bool isNull, + isPartition; + + /* + * The target tuple was concurrently updated by some other + * transaction. Run EvalPlanQual() with the new version of + * the tuple. If it does not return a tuple, then we + * switch to the NOT MATCHED list of actions. If it does + * return a tuple and the join qual is still satisfied, + * then we just need to recheck the MATCHED actions, + * starting from the top, and execute the first qualifying + * action. + */ + resultRelationDesc = resultRelInfo->ri_RelationDesc; + isPartition = RELATION_IS_PARTITIONED(resultRelationDesc); + if (isPartition) { + Partition partition = NULL; + + searchFakeReationForPartitionOid(estate->esfRelations, + estate->es_query_cxt, + resultRelationDesc, + oldPartitionOid, + GetCurrentPartitionNo(oldPartitionOid), + partRelationDesc, + partition, + RowExclusiveLock); + } + lockmode = ExecUpdateLockMode(estate, resultRelInfo); + epqslot = EvalPlanQual(estate, epqstate, + isPartition ? partRelationDesc : resultRelationDesc, + resultRelInfo->ri_RangeTableIndex, + lockmode, &tmfd.ctid, tmfd.xmax, + resultRelationDesc->rd_rel->relrowmovement); + + /* + * If we got no tuple, or the tuple we get has a + * NULL ctid, go back to caller: this one is not a + * MATCHED tuple anymore, so they can retry with + * NOT MATCHED actions. + */ + if (TupIsNull(epqslot)) + return false; + + (void) ExecGetJunkAttribute(epqslot, + resultRelInfo->ri_junkFilter->jf_junkAttNo, + &isNull); + if (isNull) + return false; + + /* + * For partitioned table we have to check if the partition oid + * is NULL. + */ + if (isPartition) { + Datum partoid; + partoid = ExecGetJunkAttribute(epqslot, + resultRelInfo->ri_partOidAttNum, + &isNull); + if (isNull) + ereport(ERROR, + (errcode(ERRCODE_NULL_JUNK_ATTRIBUTE), + errmsg("tableoid is null when merge partitioned table"))); + Assert(oldPartitionOid == DatumGetObjectId(partoid)); + } + + /* + * A non-NULL ctid means that we are still dealing + * with MATCHED case. Restart the loop so that we + * apply all the MATCHED rules again, to ensure + * that the first qualifying WHEN MATCHED action + * is executed. + * + * Update tupleid to that of the new tuple, for + * the refetch we do at the top. + */ + saved_slot = slot = epqslot; + *tupleid = tmfd.ctid; + goto lmerge_matched; + } + default: + elog(ERROR, "unexpected tuple operation result: %d", result); + break; } } if (action->commandType == CMD_UPDATE /* && tuple_updated*/) diff --git a/src/gausskernel/runtime/executor/nodeModifyTable.cpp b/src/gausskernel/runtime/executor/nodeModifyTable.cpp index 924f3b6cf..c5f9750db 100644 --- a/src/gausskernel/runtime/executor/nodeModifyTable.cpp +++ b/src/gausskernel/runtime/executor/nodeModifyTable.cpp @@ -720,10 +720,9 @@ static bool ExecConflictUpdate(ModifyTableState* mtstate, ResultRelInfo* resultR ExecProject(resultRelInfo->ri_updateProj, NULL); /* Evaluate where qual if exists, add to count if filtered */ if (ExecQual(upsertState->us_updateWhere, econtext, false)) { - TM_Result out_result; *returning = ExecUpdate(conflictTid, oldPartitionOid, bucketid, NULL, upsertState->us_updateproj, planSlot, &mtstate->mt_epqstate, - mtstate, canSetTag, ((ModifyTable*)mtstate->ps.plan)->partKeyUpdated, &out_result, partExprKeyStr); + mtstate, canSetTag, ((ModifyTable*)mtstate->ps.plan)->partKeyUpdated, NULL, partExprKeyStr); } else { InstrCountFiltered1(&mtstate->ps, 1); } @@ -2010,33 +2009,6 @@ end:; return NULL; } -/* - * check whether the tuple still match the merge condition after the tuple has been updated by other transaction. - * [in] node, executor node - * [in] epq_slot, the tuple slot after recheck on the updated tuple - * [in] mergeMatchedActionStates, update action state - * [in] fake_relation, heap table relation - * [in/out] slot, in: origin tuple slot, out: slot after merge projection - * [out] tuple, if the updated tuple still match the merge condition, return the new projected tuple - * [return value] true for match, false for not match. - */ -static bool MatchMergeCondition(ModifyTableState* node, TupleTableSlot* epq_slot, List* mergeMatchedActionStates, - Relation fake_relation, AttrNumber junkAttno, TupleTableSlot** slot, Tuple *tuple) -{ - /* resultRelInfo->ri_mergeState is always not null */ - *slot = ExecMergeProjQual(node, mergeMatchedActionStates, node->ps.ps_ExprContext, epq_slot, *slot, node->ps.state); - if (*slot != NULL) { - bool is_null = false; - /* Check after epq, whether the ctid is null. null means the updated row does not match */ - (void)ExecGetJunkAttribute(epq_slot, junkAttno, &is_null); - if (!is_null) { - *tuple = tableam_tslot_get_tuple_from_slot(fake_relation, *slot); - return true; - } - } - return false; -} - /* ---------------------------------------------------------------- * ExecUpdate * @@ -2060,7 +2032,7 @@ static bool MatchMergeCondition(ModifyTableState* node, TupleTableSlot* epq_slot TupleTableSlot* ExecUpdate(ItemPointer tupleid, Oid oldPartitionOid, /* when update a partitioned table , give a partitionOid to find the tuple */ int2 bucketid, HeapTupleHeader oldtuple, TupleTableSlot* slot, TupleTableSlot* planSlot, EPQState* epqstate, - ModifyTableState* node, bool canSetTag, bool partKeyUpdate, TM_Result* out_result, char* partExprKeyStr) + ModifyTableState* node, bool canSetTag, bool partKeyUpdate, TM_Result* uresultp, char* partExprKeyStr, TM_FailureData* tmfdp) { EState* estate = node->ps.state; Tuple tuple = NULL; @@ -2085,13 +2057,13 @@ TupleTableSlot* ExecUpdate(ItemPointer tupleid, ConflictInfoData conflictInfo; Oid conflictPartOid = InvalidOid; int2 conflictBucketid = InvalidBktId; + bool cross_partition = (partKeyUpdate || partExprKeyStr != NULL); #ifdef PGXC RemoteQueryState* result_remote_rel = NULL; #endif bool allow_update_self = (node->mt_upsert != NULL && node->mt_upsert->us_action != UPSERT_NONE) ? true : false; - *out_result = TM_Ok; /* * get information on the (current) result relation */ @@ -2146,9 +2118,9 @@ TupleTableSlot* ExecUpdate(ItemPointer tupleid, result_rel_info->ri_TrigDesc && result_rel_info->ri_TrigDesc->trig_update_before_row) { #ifdef PGXC slot = ExecBRUpdateTriggers(estate, epqstate, result_rel_info, oldPartitionOid, - bucketid, oldtuple, tupleid, slot); + bucketid, oldtuple, tupleid, slot, uresultp, tmfdp); #else - slot = ExecBRUpdateTriggers(estate, epqstate, result_rel_info, tupleid, slot); + slot = ExecBRUpdateTriggers(estate, epqstate, result_rel_info, tupleid, slot, uresultp, tmfdp); #endif if (slot == NULL) { @@ -2159,6 +2131,9 @@ TupleTableSlot* ExecUpdate(ItemPointer tupleid, // tableam /* trigger might have changed tuple */ tuple = tableam_tslot_get_tuple_from_slot(result_relation_desc, slot); + + /* The partition key might have been updated by the trigger. */ + cross_partition = true; } /* INSTEAD OF ROW UPDATE Triggers */ @@ -2246,6 +2221,8 @@ TupleTableSlot* ExecUpdate(ItemPointer tupleid, bool update_fix_result = ExecComputeStoredUpdateExpr(result_rel_info, estate, slot, tuple, CMD_UPDATE, tupleid, oldPartitionOid, bucketid); if (!update_fix_result) { tuple = slot->tts_tuple; + /* The partition key might have been updated by on update rule. */ + cross_partition = true; } } @@ -2255,6 +2232,8 @@ TupleTableSlot* ExecUpdate(ItemPointer tupleid, if (result_relation_desc->rd_att->constr && result_relation_desc->rd_att->constr->has_generated_stored) { ExecComputeStoredGenerated(result_rel_info, estate, slot, tuple, CMD_UPDATE); tuple = slot->tts_tuple; + /* The partition key might have been updated by generated expr. */ + cross_partition = true; } if (result_relation_desc->rd_att->constr) { @@ -2333,12 +2312,18 @@ TupleTableSlot* ExecUpdate(ItemPointer tupleid, estate->es_crosscheck_snapshot, estate->es_snapshot, true, // wait for commit &oldslot, &tmfd, &update_indexes, &modifiedIdxAttrs, allow_update_self, allowInplaceUpdate, &lockmode); - *out_result = result; + + /* Let the caller know about the status of this operation */ + if (uresultp) + *uresultp = result; + if (tmfdp) + *tmfdp = tmfd; + switch (result) { case TM_SelfUpdated: case TM_SelfModified: /* can not update one row more than once for merge into */ - if (node->operation == CMD_MERGE && !MEGRE_UPDATE_MULTI) { + if (node->operation == CMD_MERGE && !MERGE_UPDATE_MULTI) { ereport(ERROR, (errmodule(MOD_EXECUTOR), (errcode(ERRCODE_TOO_MANY_ROWS), @@ -2404,27 +2389,23 @@ TupleTableSlot* ExecUpdate(ItemPointer tupleid, errmsg("concurrent update under Stream mode is not yet supported"))); } + /* + * Recheck the tuple using EPQ. For MERGE, we leave this + * to the caller (it must do additional rechecking, and + * might end up executing a different action entirely). + */ + if (uresultp && estate->es_plannedstmt->commandType == CMD_MERGE) + return NULL; + TupleTableSlot *epq_slot = EvalPlanQual(estate, epqstate, fake_relation, result_rel_info->ri_RangeTableIndex, lockmode, &tmfd.ctid, tmfd.xmax, false); if (!TupIsNull(epq_slot)) { *tupleid = tmfd.ctid; - /* - * For merge into query, mergeMatchedAction's targetlist is not same as junk filter's - * targetlist. Here, epqslot is a plan slot, target table needs slot to be projected - * from plan slot. - */ - if (node->operation == CMD_MERGE) { - if (MatchMergeCondition(node, epq_slot, result_rel_info->ri_mergeState->matchedActionStates, - fake_relation, result_rel_info->ri_junkFilter->jf_junkAttNo, &slot, &tuple)) { - goto lreplace; - } - } else { - slot = ExecFilterJunk(result_rel_info->ri_junkFilter, epq_slot); + slot = ExecFilterJunk(result_rel_info->ri_junkFilter, epq_slot); - tuple = tableam_tslot_get_tuple_from_slot(fake_relation, slot); - goto lreplace; - } + tuple = tableam_tslot_get_tuple_from_slot(fake_relation, slot); + goto lreplace; } /* Updated tuple not matched; nothing to do */ @@ -2486,7 +2467,7 @@ TupleTableSlot* ExecUpdate(ItemPointer tupleid, if (partExprKeyStr) { newval = ComputePartKeyExprTuple(result_relation_desc, estate, slot, NULL, partExprKeyStr); } - if (!partExprKeyStr && !partKeyUpdate) { + if (!cross_partition) { row_movement = false; new_partId = oldPartitionOid; } else { @@ -2649,12 +2630,18 @@ TupleTableSlot* ExecUpdate(ItemPointer tupleid, allow_update_self, allowInplaceUpdate, &lockmode); - *out_result = result; + + /* Let the caller know about the status of this operation */ + if (uresultp) + *uresultp = result; + if (tmfdp) + *tmfdp = tmfd; + switch (result) { case TM_SelfUpdated: case TM_SelfModified: /* can not update one row more than once for merge into */ - if (node->operation == CMD_MERGE && !MEGRE_UPDATE_MULTI) { + if (node->operation == CMD_MERGE && !MERGE_UPDATE_MULTI) { ereport(ERROR, (errmodule(MOD_EXECUTOR), (errcode(ERRCODE_TOO_MANY_ROWS), errmsg("unable to get a stable set of rows in the source tables")))); } @@ -2707,6 +2694,15 @@ TupleTableSlot* ExecUpdate(ItemPointer tupleid, errmsg("concurrent update under Stream mode is not yet supported"))); } + + /* + * Recheck the tuple using EPQ. For MERGE, we leave this + * to the caller (it must do additional rechecking, and + * might end up executing a different action entirely). + */ + if (uresultp && estate->es_plannedstmt->commandType == CMD_MERGE) + return NULL; + TupleTableSlot *epq_slot = EvalPlanQual(estate, epqstate, fake_relation, result_rel_info->ri_RangeTableIndex, lockmode, &tmfd.ctid, tmfd.xmax, result_relation_desc->rd_rel->relrowmovement); @@ -2714,22 +2710,10 @@ TupleTableSlot* ExecUpdate(ItemPointer tupleid, if (!TupIsNull(epq_slot)) { *tupleid = tmfd.ctid; - /* - * For merge into query, mergeMatchedAction's targetlist is not same as junk - * filter's targetlist. Here, epq_slot is a plan slot, target table needs slot to be - * projected from plan slot. - */ - if (node->operation == CMD_MERGE) { - if (MatchMergeCondition(node, epq_slot, result_rel_info->ri_mergeState->matchedActionStates, - fake_relation, result_rel_info->ri_junkFilter->jf_junkAttNo, &slot, &tuple)) { - goto lreplace; - } - } else { - slot = ExecFilterJunk(result_rel_info->ri_junkFilter, epq_slot); + slot = ExecFilterJunk(result_rel_info->ri_junkFilter, epq_slot); - tuple = tableam_tslot_get_tuple_from_slot(fake_relation, slot); - goto lreplace; - } + tuple = tableam_tslot_get_tuple_from_slot(fake_relation, slot); + goto lreplace; } /* Updated tuple not matched; nothing to do */ @@ -2857,12 +2841,18 @@ TupleTableSlot* ExecUpdate(ItemPointer tupleid, &oldslot, &tmfd, allow_update_self); - *out_result = result; + + /* Let the caller know about the status of this operation */ + if (uresultp) + *uresultp = result; + if (tmfdp) + *tmfdp = tmfd; + switch (result) { case TM_SelfUpdated: case TM_SelfModified: /* can not update one row more than once for merge into */ - if (node->operation == CMD_MERGE && !MEGRE_UPDATE_MULTI) { + if (node->operation == CMD_MERGE && !MERGE_UPDATE_MULTI) { ereport(ERROR, (errmodule(MOD_EXECUTOR), (errcode(ERRCODE_TOO_MANY_ROWS), errmsg("unable to get a stable set of rows in the source tables")))); } @@ -2939,6 +2929,14 @@ TupleTableSlot* ExecUpdate(ItemPointer tupleid, errmsg("concurrent update under Stream mode is not yet supported"))); } + /* + * Recheck the tuple using EPQ. For MERGE, we leave this + * to the caller (it must do additional rechecking, and + * might end up executing a different action entirely). + */ + if (uresultp && estate->es_plannedstmt->commandType == CMD_MERGE) + return NULL; + TupleTableSlot* epq_slot = EvalPlanQual(estate, epqstate, old_fake_relation, @@ -2950,22 +2948,10 @@ TupleTableSlot* ExecUpdate(ItemPointer tupleid, /* Try to fetch latest tuple values in row movement case */ if (!TupIsNull(epq_slot)) { *tupleid = tmfd.ctid; - /* - * For merge into query, mergeMatchedAction's targetlist is not same as - * junk filter's targetlist. Here, epqslot is a plan slot, target table - * needs slot to be projected from plan slot. - */ - if (node->operation == CMD_MERGE) { - if (MatchMergeCondition(node, epq_slot, result_rel_info->ri_mergeState->matchedActionStates, - old_fake_relation, result_rel_info->ri_junkFilter->jf_junkAttNo, &slot, &tuple)) { - goto ldelete; - } - } else { - slot = ExecFilterJunk(result_rel_info->ri_junkFilter, epq_slot); - - tuple = tableam_tslot_get_tuple_from_slot(old_fake_relation, slot); - goto ldelete; - } + slot = ExecFilterJunk(result_rel_info->ri_junkFilter, epq_slot); + + tuple = tableam_tslot_get_tuple_from_slot(old_fake_relation, slot); + goto ldelete; } /* Updated tuple not matched; nothing to do */ @@ -3721,7 +3707,6 @@ static TupleTableSlot* ExecModifyTable(PlanState* state) } break; case CMD_UPDATE: { - TM_Result out_result; slot = ExecUpdate(tuple_id, old_partition_oid, bucketid, @@ -3732,7 +3717,7 @@ static TupleTableSlot* ExecModifyTable(PlanState* state) node, node->canSetTag, part_key_updated, - &out_result, + NULL, partExprKeyStr); } break; case CMD_DELETE: diff --git a/src/include/commands/trigger.h b/src/include/commands/trigger.h index f475272a6..5ec192602 100644 --- a/src/include/commands/trigger.h +++ b/src/include/commands/trigger.h @@ -146,7 +146,7 @@ extern TupleTableSlot* ExecBRUpdateTriggers(EState* estate, EPQState* epqstate, #ifdef PGXC HeapTupleHeader datanode_tuphead, #endif - ItemPointer tupleid, TupleTableSlot* slot); + ItemPointer tupleid, TupleTableSlot* slot, TM_Result* result = NULL, TM_FailureData* tmfd = NULL); extern void ExecARUpdateTriggers(EState* estate, ResultRelInfo* relinfo, Oid oldPartitionOid, int2 bucketid, Oid newPartitionOid, ItemPointer tupleid, HeapTuple newtuple, #ifdef PGXC @@ -201,6 +201,7 @@ extern void InvalidRelcacheForTriggerFunction(Oid funcoid, Oid returnType); extern void ResetTrigShipFlag(); extern HeapTuple GetTupleForTrigger(EState* estate, EPQState* epqstate, ResultRelInfo* relinfo, Oid targetPartitionOid, - int2 bucketid, ItemPointer tid, LockTupleMode lockmode, TupleTableSlot** newSlot); + int2 bucketid, ItemPointer tid, LockTupleMode lockmode, TupleTableSlot** newSlot, TM_Result* result = NULL, + TM_FailureData* tmfd = NULL); #endif /* TRIGGER_H */ diff --git a/src/include/executor/node/nodeModifyTable.h b/src/include/executor/node/nodeModifyTable.h index 99e6a6fa6..3fe98835a 100644 --- a/src/include/executor/node/nodeModifyTable.h +++ b/src/include/executor/node/nodeModifyTable.h @@ -46,7 +46,7 @@ extern TupleTableSlot* ExecDelete(ItemPointer tupleid, Oid deletePartitionOid, i extern TupleTableSlot* ExecUpdate(ItemPointer tupleid, Oid oldPartitionOid, int2 bucketid, HeapTupleHeader oldtuple, TupleTableSlot* slot, TupleTableSlot* planSlot, EPQState* epqstate, ModifyTableState* node, bool canSetTag, - bool partKeyUpdate, TM_Result* out_result, char* partExprKeyStr = NULL); + bool partKeyUpdate, TM_Result* uresultp = NULL, char* partExprKeyStr = NULL, TM_FailureData* tmfd = NULL); template extern TupleTableSlot* ExecInsertT(ModifyTableState* state, TupleTableSlot* slot, TupleTableSlot* planSlot, diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h index a52ef3218..363bf6711 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h @@ -181,7 +181,7 @@ extern void SSUpgradeFileBeforeCommit(); #define OPT_UNBIND_DIVIDE_BOUND 64 #define OPT_CORRECT_TO_NUMBER 128 #define OPT_CONCAT_VARIADIC 256 -#define OPT_MEGRE_UPDATE_MULTI 512 +#define OPT_MERGE_UPDATE_MULTI 512 #define OPT_CONVERT_TO_NUMERIC 1024 #define OPT_PLSTMT_IMPLICIT_SAVEPOINT 2048 #define OPT_HIDE_TAILING_ZERO 4096 @@ -219,7 +219,7 @@ extern void SSUpgradeFileBeforeCommit(); * option is blank and the behavior is new and compatible with current A and C mode, if the option is set, the * behavior is old and the same as previous GAUSSDB kernel. */ #define CONCAT_VARIADIC (!(u_sess->utils_cxt.behavior_compat_flags & OPT_CONCAT_VARIADIC)) -#define MEGRE_UPDATE_MULTI (u_sess->utils_cxt.behavior_compat_flags & OPT_MEGRE_UPDATE_MULTI) +#define MERGE_UPDATE_MULTI (u_sess->utils_cxt.behavior_compat_flags & OPT_MERGE_UPDATE_MULTI) #define CONVERT_STRING_DIGIT_TO_NUMERIC (u_sess->utils_cxt.behavior_compat_flags & OPT_CONVERT_TO_NUMERIC) #define PLSTMT_IMPLICIT_SAVEPOINT (u_sess->utils_cxt.behavior_compat_flags & OPT_PLSTMT_IMPLICIT_SAVEPOINT) #define HIDE_TAILING_ZERO (u_sess->utils_cxt.behavior_compat_flags & OPT_HIDE_TAILING_ZERO) diff --git a/src/test/regress/expected/merge_into_deleted.out b/src/test/regress/expected/merge_into_deleted.out new file mode 100644 index 000000000..6c8786aca --- /dev/null +++ b/src/test/regress/expected/merge_into_deleted.out @@ -0,0 +1,53 @@ +create schema merge_into_deleted; +set search_path = 'merge_into_deleted'; +create table t1 (c1 int, c2 text, c3 timestamp); +insert into t1 values (1, 'a', '2023-09-15'); +-- concurrently deleted and insert, do update and insert +\parallel on 2 +begin + delete from t1 where c1 = 1; + perform pg_sleep(3); +end; +/ +begin + perform pg_sleep(1); + insert into t1 values (1, 'b', '2023-09-16'); + merge into t1 using (select 1 c1) t2 + on (t1.c1 = t2.c1) + when matched then update set c2 = 'c' + when not matched then insert values (2, 'c', '2023-09-17'); +end; +/ +\parallel off +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 order by c1; + c1 | c2 | to_char +----+----+------------ + 1 | c | 2023-09-16 + 2 | c | 2023-09-17 +(2 rows) + +-- concurrently deleted, not matched, do insert +\parallel on 2 +begin + delete from t1 where c1 = 1; + perform pg_sleep(3); +end; +/ +begin + perform pg_sleep(1); + merge into t1 using (select 1 c1) t2 + on (t1.c1 = t2.c1) + when matched then update set c2 = 'd' + when not matched then insert values (3, 'd', '2023-09-18'); +end; +/ +\parallel off +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 order by c1; + c1 | c2 | to_char +----+----+------------ + 2 | c | 2023-09-17 + 3 | d | 2023-09-18 +(2 rows) + +drop schema merge_into_deleted cascade; +NOTICE: drop cascades to table t1 diff --git a/src/test/regress/expected/merge_into_partition_row_movement.out b/src/test/regress/expected/merge_into_partition_row_movement.out new file mode 100644 index 000000000..3af3d3754 --- /dev/null +++ b/src/test/regress/expected/merge_into_partition_row_movement.out @@ -0,0 +1,267 @@ +create schema merge_into_partition_row_movement; +set search_path = 'merge_into_partition_row_movement'; +create table t1 (c1 int, c2 text, c3 timestamp) +partition by range(c1) +( + partition p1 values less than (100), + partition p2 values less than (200) +); +create table t2 (c1 int); +insert into t1 values (1, 'a', '2023-09-15'), (101, 'b', '2023-09-16'); +insert into t2 values (2), (102); +begin + update t1 set c1 = 102 where c1 = 1; + update t1 set c1 = 2 where c1 = 101; + merge into t1 using t2 + on (t1.c1 = t2.c1) + when matched then update set c2 = 'd' + when not matched then insert values (3, 'd', '2023-09-18'); +end; +/ +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 partition (p1) order by c1; + c1 | c2 | to_char +----+----+------------ + 2 | d | 2023-09-16 +(1 row) + +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 partition (p2) order by c1; + c1 | c2 | to_char +-----+----+------------ + 102 | d | 2023-09-15 +(1 row) + +delete from t1; +insert into t1 values (1, 'a', '2023-09-15'), (101, 'b', '2023-09-16'); +delete from t2; +insert into t2 values (1), (2); +-- success, concurrently update +\parallel on 2 +begin + update t1 set c1 = 2 where c1 = 1; + update t1 set c1 = 1 where c1 = 101; + perform pg_sleep(3); +end; +/ +begin + perform pg_sleep(1); + merge into t1 using t2 + on (t1.c1 = t2.c1) + when matched then update set c2 = 'd' + when not matched then insert values (3, 'd', '2023-09-18'); +end; +/ +\parallel off +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 partition (p1) order by c1; + c1 | c2 | to_char +----+----+------------ + 1 | b | 2023-09-16 + 2 | a | 2023-09-15 + 3 | d | 2023-09-18 + 3 | d | 2023-09-18 +(4 rows) + +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 partition (p2) order by c1; + c1 | c2 | to_char +----+----+--------- +(0 rows) + +delete from t1; +insert into t1 values (1, 'a', '2023-09-15'), (101, 'b', '2023-09-16'); +delete from t2; +insert into t2 values (1), (101); +-- error +\parallel on 2 +begin + update t1 set c1 = 102 where c1 = 1; + update t1 set c1 = 2 where c1 = 101; + perform pg_sleep(3); +end; +/ +begin + perform pg_sleep(1); + merge into t1 using t2 + on (t1.c1 = t2.c1) + when matched then update set c2 = 'd' + when not matched then insert values (3, 'd', '2023-09-18'); +end; +/ +\parallel off +ERROR: partition table update conflict +DETAIL: disable row movement of table can avoid this conflict +CONTEXT: SQL statement "merge into t1 using t2 + on (t1.c1 = t2.c1) + when matched then update set c2 = 'd' + when not matched then insert values (3, 'd', '2023-09-18')" +PL/pgSQL function inline_code_block line 3 at SQL statement +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 partition (p1) order by c1; + c1 | c2 | to_char +----+----+------------ + 2 | b | 2023-09-16 +(1 row) + +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 partition (p2) order by c1; + c1 | c2 | to_char +-----+----+------------ + 102 | a | 2023-09-15 +(1 row) + +delete from t1; +insert into t1 values (1, 'a', '2023-09-15'), (101, 'b', '2023-09-16'); +delete from t2; +insert into t2 values (1), (2); +create or replace function t1_tri_func() return trigger as +begin + if (old.c1 < 101) then + new.c1 = 150; + else + new.c1 = 50; + end if; + return new; +end; +/ +create trigger t1_tri + before update on t1 + for each row + execute procedure t1_tri_func(); +-- error +\parallel on 2 +begin + update t1 set c1 = 2 where c1 = 1; + update t1 set c1 = 2 where c1 = 101; + perform pg_sleep(3); +end; +/ +begin + perform pg_sleep(1); + merge into t1 using t2 + on (t1.c1 = t2.c1) + when matched then update set c2 = 'd' + when not matched then insert values (4, 'e', '2023-09-18'); +end; +/ +\parallel off +ERROR: partition table update conflict +DETAIL: disable row movement of table can avoid this conflict +CONTEXT: SQL statement "merge into t1 using t2 + on (t1.c1 = t2.c1) + when matched then update set c2 = 'd' + when not matched then insert values (4, 'e', '2023-09-18')" +PL/pgSQL function inline_code_block line 3 at SQL statement +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 partition (p1) order by c1; + c1 | c2 | to_char +----+----+------------ + 50 | b | 2023-09-16 +(1 row) + +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 partition (p2) order by c1; + c1 | c2 | to_char +-----+----+------------ + 150 | a | 2023-09-15 +(1 row) + +delete from t1; +insert into t1 values (1, 'a', '2023-09-15'), (101, 'b', '2023-09-16'); +delete from t2; +insert into t2 values (1), (101); +-- error +\parallel on 2 +begin + update t1 set c3 = '2023-09-19' where c1 = 1; + update t1 set c3 = '2023-09-20' where c1 = 101; + perform pg_sleep(3); +end; +/ +begin + perform pg_sleep(1); + merge into t1 using t2 + on (t1.c1 = t2.c1) + when matched then update set c2 = 'd' + when not matched then insert values (4, 'e', '2023-09-18'); +end; +/ +\parallel off +ERROR: partition table update conflict +DETAIL: disable row movement of table can avoid this conflict +CONTEXT: SQL statement "merge into t1 using t2 + on (t1.c1 = t2.c1) + when matched then update set c2 = 'd' + when not matched then insert values (4, 'e', '2023-09-18')" +PL/pgSQL function inline_code_block line 3 at SQL statement +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 partition (p1) order by c1; + c1 | c2 | to_char +----+----+------------ + 50 | b | 2023-09-20 +(1 row) + +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 partition (p2) order by c1; + c1 | c2 | to_char +-----+----+------------ + 150 | a | 2023-09-19 +(1 row) + +delete from t1; +insert into t1 values (1, 'a', '2023-09-15'), (101, 'b', '2023-09-16'); +delete from t2; +insert into t2 values (1), (101); +begin + merge into t1 using t2 + on (t1.c1 = t2.c1) + when matched then update set c2 = 'd' + when not matched then insert values (4, 'e', '2023-09-18'); +end; +/ +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 partition (p1) order by c1; + c1 | c2 | to_char +----+----+------------ + 50 | d | 2023-09-16 +(1 row) + +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 partition (p2) order by c1; + c1 | c2 | to_char +-----+----+------------ + 150 | d | 2023-09-15 +(1 row) + +delete from t1; +insert into t1 values (1, 'a', '2023-09-15'), (101, 'b', '2023-09-16'); +delete from t2; +insert into t2 values (1), (101); +-- error +\parallel on 2 +begin + delete from t1 where c1 = 1; + perform pg_sleep(3); +end; +/ +begin + perform pg_sleep(1); + merge into t1 using t2 + on (t1.c1 = t2.c1) + when matched then update set c2 = 'd' + when not matched then insert values (4, 'e', '2023-09-18'); +end; +/ +\parallel off +ERROR: partition table update conflict +DETAIL: disable row movement of table can avoid this conflict +CONTEXT: SQL statement "merge into t1 using t2 + on (t1.c1 = t2.c1) + when matched then update set c2 = 'd' + when not matched then insert values (4, 'e', '2023-09-18')" +PL/pgSQL function inline_code_block line 3 at SQL statement +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 partition (p1) order by c1; + c1 | c2 | to_char +----+----+--------- +(0 rows) + +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 partition (p2) order by c1; + c1 | c2 | to_char +-----+----+------------ + 101 | b | 2023-09-16 +(1 row) + +drop schema merge_into_partition_row_movement cascade; +NOTICE: drop cascades to 3 other objects +DETAIL: drop cascades to table t1 +drop cascades to table t2 +drop cascades to function t1_tri_func() diff --git a/src/test/regress/expected/merge_into_selfmodified.out b/src/test/regress/expected/merge_into_selfmodified.out new file mode 100644 index 000000000..78cfa7145 --- /dev/null +++ b/src/test/regress/expected/merge_into_selfmodified.out @@ -0,0 +1,133 @@ +create schema merge_into_selfmodified; +set search_path = 'merge_into_selfmodified'; +create table t1 (c1 int, c2 text, c3 timestamp); +create table t2 (c1 int); +insert into t1 values (1, 'a', '2023-09-15'); +create or replace function t1_tri_func() return trigger as +begin + new.c3 = '2023-09-16'; + return new; +end; +/ +create trigger t1_tri + before update on t1 + for each row + execute procedure t1_tri_func(); +-- success, t2 is null, nothing to do +merge into t1 using t2 +on (t1.c1 = t2.c1) +when matched then update set c2 = 'b' +when not matched then insert values (2, 'b', '2023-09-17'); +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 order by c1; + c1 | c2 | to_char +----+----+------------ + 1 | a | 2023-09-15 +(1 row) + +-- success, t2 has one row, not matched, do insert +merge into t1 using (select null c1) t2 +on (t1.c1 = t2.c1) +when matched then update set c2 = 'c' +when not matched then insert values (3, 'c', '2023-09-18'); +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 order by c1; + c1 | c2 | to_char +----+----+------------ + 1 | a | 2023-09-15 + 3 | c | 2023-09-18 +(2 rows) + +insert into t2 values (1); +-- success, matched, do update +merge into t1 using t2 +on (t1.c1 = t2.c1) +when matched then update set c2 = 'd' +when not matched then insert values (4, 'd', '2023-09-19'); +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 order by c1; + c1 | c2 | to_char +----+----+------------ + 1 | d | 2023-09-16 + 3 | c | 2023-09-18 +(2 rows) + +insert into t2 values (1); +-- error, affect one row a second time +merge into t1 using t2 +on (t1.c1 = t2.c1) +when matched then update set c2 = 'e' +when not matched then insert values (5, 'e', '2023-09-20'); +ERROR: MERGE command cannot affect row a second time +HINT: Ensure that not more than one source row matches any one target row. +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 order by c1; + c1 | c2 | to_char +----+----+------------ + 1 | d | 2023-09-16 + 3 | c | 2023-09-18 +(2 rows) + +set behavior_compat_options = 'merge_update_multi'; +-- success, but update only once +merge into t1 using t2 +on (t1.c1 = t2.c1) +when matched then update set c2 = 'f' +when not matched then insert values (6, 'f', '2023-09-21'); +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 order by c1; + c1 | c2 | to_char +----+----+------------ + 1 | f | 2023-09-16 + 3 | c | 2023-09-18 +(2 rows) + +insert into t2 values (7); +-- success, do update and insert +merge into t1 using t2 +on (t1.c1 = t2.c1) +when matched then update set c2 = 'g' +when not matched then insert values (7, 'g', '2023-09-22'); +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 order by c1; + c1 | c2 | to_char +----+----+------------ + 1 | g | 2023-09-16 + 3 | c | 2023-09-18 + 7 | g | 2023-09-22 +(3 rows) + +insert into t2 values (8),(8); +-- success, do update and insert only once +merge into t1 using t2 +on (t1.c1 = t2.c1) +when matched then update set c2 = 'h' +when not matched then insert values (8, 'h', '2023-09-23'); +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 order by c1; + c1 | c2 | to_char +----+----+------------ + 1 | h | 2023-09-16 + 3 | c | 2023-09-18 + 7 | h | 2023-09-16 + 8 | h | 2023-09-23 + 8 | h | 2023-09-23 +(5 rows) + +insert into t2 values (9),(10); +-- success, do update and insert only once +merge into t1 using t2 +on (t1.c1 = t2.c1) +when matched then update set c2 = 'i' +when not matched then insert values (9, 'i', '2023-09-24'); +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 order by c1; + c1 | c2 | to_char +----+----+------------ + 1 | i | 2023-09-16 + 3 | c | 2023-09-18 + 7 | i | 2023-09-16 + 8 | i | 2023-09-16 + 8 | i | 2023-09-16 + 9 | i | 2023-09-24 + 9 | i | 2023-09-24 +(7 rows) + +reset behavior_compat_options; +drop schema merge_into_selfmodified cascade; +NOTICE: drop cascades to 3 other objects +DETAIL: drop cascades to table t1 +drop cascades to table t2 +drop cascades to function t1_tri_func() diff --git a/src/test/regress/expected/merge_into_updated.out b/src/test/regress/expected/merge_into_updated.out new file mode 100644 index 000000000..993e20df3 --- /dev/null +++ b/src/test/regress/expected/merge_into_updated.out @@ -0,0 +1,140 @@ +create schema merge_into_updated; +set search_path = 'merge_into_updated'; +create table t1 (c1 int, c2 text, c3 timestamp); +insert into t1 values (1, 'a', '2023-09-15'); +create or replace function t1_tri_func() return trigger as +begin + new.c3 = '2023-09-16'; + return new; +end; +/ +create trigger t1_tri + before update on t1 + for each row + execute procedure t1_tri_func(); +-- success, matched, do update +\parallel on 2 +begin + update t1 set c2 = 'b' where c1 = 1; + perform pg_sleep(3); +end; +/ +begin + perform pg_sleep(1); + merge into t1 using (select 1 c1) t2 + on (t1.c1 = t2.c1) + when matched then update set c2 = 'c' + when not matched then insert values (2, 'c', '2023-09-17'); +end; +/ +\parallel off +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 order by c1; + c1 | c2 | to_char +----+----+------------ + 1 | c | 2023-09-16 +(1 row) + +-- success, matched, do update +\parallel on 2 +begin + update t1 set c2 = 'b' where c1 = 1; + perform pg_sleep(3); +end; +/ +begin + perform pg_sleep(1); + insert into t1 values (1, 'hello', '2023-09-17'); + delete from t1 where c2 = 'hello'; + merge into t1 using (select 1 c1) t2 + on (t1.c1 = t2.c1) + when matched then update set c2 = 'd' + when not matched then insert values (2, 'c', '2023-09-17'); +end; +/ +\parallel off +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 order by c1; + c1 | c2 | to_char +----+----+------------ + 1 | d | 2023-09-16 +(1 row) + +-- success, concurrently update join condition, not matched, do insert +\parallel on 2 +begin + update t1 set c1 = 2 where c1 = 1; + perform pg_sleep(3); +end; +/ +begin + perform pg_sleep(1); + merge into t1 using (select 1 c1) t2 + on (t1.c1 = t2.c1) + when matched then update set c2 = 'd' + when not matched then insert values (3, 'd', '2023-09-18'); +end; +/ +\parallel off +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 order by c1; + c1 | c2 | to_char +----+----+------------ + 2 | d | 2023-09-16 + 3 | d | 2023-09-18 +(2 rows) + +-- success, not matched, do insert +\parallel on 2 +begin + update t1 set c2 = 'b'; + perform pg_sleep(3); +end; +/ +begin + perform pg_sleep(1); + insert into t1 values (1, 'hello', '2023-09-17'); + delete from t1; + merge into t1 using (select 1 c1) t2 + on (t1.c1 = t2.c1) + when matched then update set c2 = 'c' + when not matched then insert values (2, 'c', '2023-09-17'); +end; +/ +\parallel off +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 order by c1; + c1 | c2 | to_char +----+----+------------ + 2 | c | 2023-09-17 +(1 row) + +-- trigger update the join condition, not matched, do insert +create or replace function t1_tri_func() return trigger as +begin + new.c1 = 4; + return new; +end; +/ +\parallel on 2 +begin + update t1 set c2 = 'b'; + perform pg_sleep(3); +end; +/ +begin + perform pg_sleep(1); + merge into t1 using (select 2 c1) t2 + on (t1.c1 = t2.c1) + when matched then update set c2 = 'e' + when not matched then insert values (4, 'e', '2023-09-19'); +end; +/ +\parallel off +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 order by c1; + c1 | c2 | to_char +----+----+------------ + 4 | b | 2023-09-17 + 4 | e | 2023-09-19 +(2 rows) + +drop schema merge_into_updated cascade; +NOTICE: drop cascades to 2 other objects +DETAIL: drop cascades to table t1 +drop cascades to function t1_tri_func() diff --git a/src/test/regress/parallel_schedule0 b/src/test/regress/parallel_schedule0 index 6935231d7..098e0b741 100644 --- a/src/test/regress/parallel_schedule0 +++ b/src/test/regress/parallel_schedule0 @@ -277,7 +277,7 @@ test: single_node_forbidden test: single_node_mergeinto merge_subquery merge_subquery3 merge_1 test: merge_where_col -test: merge_concurrent_update_delete_1 merge_concurrent_update_delete_2 merge_concurrent_update_delete_3 +test: merge_concurrent_update_delete_1 merge_concurrent_update_delete_2 merge_concurrent_update_delete_3 merge_into_deleted merge_into_partition_row_movement merge_into_selfmodified merge_into_updated # Trigger tests test: single_node_triggers diff --git a/src/test/regress/parallel_schedule0A b/src/test/regress/parallel_schedule0A index 76da7e9e0..106f8edc9 100644 --- a/src/test/regress/parallel_schedule0A +++ b/src/test/regress/parallel_schedule0A @@ -269,7 +269,7 @@ test: single_node_forbidden test: single_node_mergeinto merge_subquery merge_subquery3 merge_1 test: merge_where_col -test: merge_concurrent_update_delete_1 merge_concurrent_update_delete_2 merge_concurrent_update_delete_3 +test: merge_concurrent_update_delete_1 merge_concurrent_update_delete_2 merge_concurrent_update_delete_3 merge_into_deleted merge_into_partition_row_movement merge_into_selfmodified merge_into_updated # Trigger tests test: single_node_triggers diff --git a/src/test/regress/sql/merge_into_deleted.sql b/src/test/regress/sql/merge_into_deleted.sql new file mode 100644 index 000000000..1432868bf --- /dev/null +++ b/src/test/regress/sql/merge_into_deleted.sql @@ -0,0 +1,44 @@ +create schema merge_into_deleted; +set search_path = 'merge_into_deleted'; + +create table t1 (c1 int, c2 text, c3 timestamp); +insert into t1 values (1, 'a', '2023-09-15'); + +-- concurrently deleted and insert, do update and insert +\parallel on 2 +begin + delete from t1 where c1 = 1; + perform pg_sleep(3); +end; +/ +begin + perform pg_sleep(1); + insert into t1 values (1, 'b', '2023-09-16'); + merge into t1 using (select 1 c1) t2 + on (t1.c1 = t2.c1) + when matched then update set c2 = 'c' + when not matched then insert values (2, 'c', '2023-09-17'); +end; +/ +\parallel off +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 order by c1; + +-- concurrently deleted, not matched, do insert +\parallel on 2 +begin + delete from t1 where c1 = 1; + perform pg_sleep(3); +end; +/ +begin + perform pg_sleep(1); + merge into t1 using (select 1 c1) t2 + on (t1.c1 = t2.c1) + when matched then update set c2 = 'd' + when not matched then insert values (3, 'd', '2023-09-18'); +end; +/ +\parallel off +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 order by c1; + +drop schema merge_into_deleted cascade; diff --git a/src/test/regress/sql/merge_into_partition_row_movement.sql b/src/test/regress/sql/merge_into_partition_row_movement.sql new file mode 100644 index 000000000..82dad97eb --- /dev/null +++ b/src/test/regress/sql/merge_into_partition_row_movement.sql @@ -0,0 +1,182 @@ +create schema merge_into_partition_row_movement; +set search_path = 'merge_into_partition_row_movement'; + +create table t1 (c1 int, c2 text, c3 timestamp) +partition by range(c1) +( + partition p1 values less than (100), + partition p2 values less than (200) +); +create table t2 (c1 int); + +insert into t1 values (1, 'a', '2023-09-15'), (101, 'b', '2023-09-16'); +insert into t2 values (2), (102); + +begin + update t1 set c1 = 102 where c1 = 1; + update t1 set c1 = 2 where c1 = 101; + merge into t1 using t2 + on (t1.c1 = t2.c1) + when matched then update set c2 = 'd' + when not matched then insert values (3, 'd', '2023-09-18'); +end; +/ +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 partition (p1) order by c1; +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 partition (p2) order by c1; + +delete from t1; +insert into t1 values (1, 'a', '2023-09-15'), (101, 'b', '2023-09-16'); +delete from t2; +insert into t2 values (1), (2); + +-- success, concurrently update +\parallel on 2 +begin + update t1 set c1 = 2 where c1 = 1; + update t1 set c1 = 1 where c1 = 101; + perform pg_sleep(3); +end; +/ +begin + perform pg_sleep(1); + merge into t1 using t2 + on (t1.c1 = t2.c1) + when matched then update set c2 = 'd' + when not matched then insert values (3, 'd', '2023-09-18'); +end; +/ +\parallel off +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 partition (p1) order by c1; +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 partition (p2) order by c1; + +delete from t1; +insert into t1 values (1, 'a', '2023-09-15'), (101, 'b', '2023-09-16'); +delete from t2; +insert into t2 values (1), (101); + +-- error +\parallel on 2 +begin + update t1 set c1 = 102 where c1 = 1; + update t1 set c1 = 2 where c1 = 101; + perform pg_sleep(3); +end; +/ +begin + perform pg_sleep(1); + merge into t1 using t2 + on (t1.c1 = t2.c1) + when matched then update set c2 = 'd' + when not matched then insert values (3, 'd', '2023-09-18'); +end; +/ +\parallel off +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 partition (p1) order by c1; +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 partition (p2) order by c1; + +delete from t1; +insert into t1 values (1, 'a', '2023-09-15'), (101, 'b', '2023-09-16'); +delete from t2; +insert into t2 values (1), (2); + +create or replace function t1_tri_func() return trigger as +begin + if (old.c1 < 101) then + new.c1 = 150; + else + new.c1 = 50; + end if; + return new; +end; +/ +create trigger t1_tri + before update on t1 + for each row + execute procedure t1_tri_func(); + +-- error +\parallel on 2 +begin + update t1 set c1 = 2 where c1 = 1; + update t1 set c1 = 2 where c1 = 101; + perform pg_sleep(3); +end; +/ +begin + perform pg_sleep(1); + merge into t1 using t2 + on (t1.c1 = t2.c1) + when matched then update set c2 = 'd' + when not matched then insert values (4, 'e', '2023-09-18'); +end; +/ +\parallel off +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 partition (p1) order by c1; +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 partition (p2) order by c1; + +delete from t1; +insert into t1 values (1, 'a', '2023-09-15'), (101, 'b', '2023-09-16'); +delete from t2; +insert into t2 values (1), (101); + +-- error +\parallel on 2 +begin + update t1 set c3 = '2023-09-19' where c1 = 1; + update t1 set c3 = '2023-09-20' where c1 = 101; + perform pg_sleep(3); +end; +/ +begin + perform pg_sleep(1); + merge into t1 using t2 + on (t1.c1 = t2.c1) + when matched then update set c2 = 'd' + when not matched then insert values (4, 'e', '2023-09-18'); +end; +/ +\parallel off +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 partition (p1) order by c1; +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 partition (p2) order by c1; + + +delete from t1; +insert into t1 values (1, 'a', '2023-09-15'), (101, 'b', '2023-09-16'); +delete from t2; +insert into t2 values (1), (101); + +begin + merge into t1 using t2 + on (t1.c1 = t2.c1) + when matched then update set c2 = 'd' + when not matched then insert values (4, 'e', '2023-09-18'); +end; +/ +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 partition (p1) order by c1; +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 partition (p2) order by c1; + +delete from t1; +insert into t1 values (1, 'a', '2023-09-15'), (101, 'b', '2023-09-16'); +delete from t2; +insert into t2 values (1), (101); + +-- error +\parallel on 2 +begin + delete from t1 where c1 = 1; + perform pg_sleep(3); +end; +/ +begin + perform pg_sleep(1); + merge into t1 using t2 + on (t1.c1 = t2.c1) + when matched then update set c2 = 'd' + when not matched then insert values (4, 'e', '2023-09-18'); +end; +/ +\parallel off +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 partition (p1) order by c1; +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 partition (p2) order by c1; + +drop schema merge_into_partition_row_movement cascade; diff --git a/src/test/regress/sql/merge_into_selfmodified.sql b/src/test/regress/sql/merge_into_selfmodified.sql new file mode 100644 index 000000000..7f2d44b8f --- /dev/null +++ b/src/test/regress/sql/merge_into_selfmodified.sql @@ -0,0 +1,91 @@ +create schema merge_into_selfmodified; +set search_path = 'merge_into_selfmodified'; + +create table t1 (c1 int, c2 text, c3 timestamp); +create table t2 (c1 int); +insert into t1 values (1, 'a', '2023-09-15'); + +create or replace function t1_tri_func() return trigger as +begin + new.c3 = '2023-09-16'; + return new; +end; +/ +create trigger t1_tri + before update on t1 + for each row + execute procedure t1_tri_func(); + +-- success, t2 is null, nothing to do +merge into t1 using t2 +on (t1.c1 = t2.c1) +when matched then update set c2 = 'b' +when not matched then insert values (2, 'b', '2023-09-17'); + +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 order by c1; + +-- success, t2 has one row, not matched, do insert +merge into t1 using (select null c1) t2 +on (t1.c1 = t2.c1) +when matched then update set c2 = 'c' +when not matched then insert values (3, 'c', '2023-09-18'); + +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 order by c1; + +insert into t2 values (1); +-- success, matched, do update +merge into t1 using t2 +on (t1.c1 = t2.c1) +when matched then update set c2 = 'd' +when not matched then insert values (4, 'd', '2023-09-19'); + +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 order by c1; + +insert into t2 values (1); +-- error, affect one row a second time +merge into t1 using t2 +on (t1.c1 = t2.c1) +when matched then update set c2 = 'e' +when not matched then insert values (5, 'e', '2023-09-20'); + +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 order by c1; + +set behavior_compat_options = 'merge_update_multi'; +-- success, but update only once +merge into t1 using t2 +on (t1.c1 = t2.c1) +when matched then update set c2 = 'f' +when not matched then insert values (6, 'f', '2023-09-21'); + +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 order by c1; + +insert into t2 values (7); +-- success, do update and insert +merge into t1 using t2 +on (t1.c1 = t2.c1) +when matched then update set c2 = 'g' +when not matched then insert values (7, 'g', '2023-09-22'); + +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 order by c1; + +insert into t2 values (8),(8); +-- success, do update and insert only once +merge into t1 using t2 +on (t1.c1 = t2.c1) +when matched then update set c2 = 'h' +when not matched then insert values (8, 'h', '2023-09-23'); + +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 order by c1; + +insert into t2 values (9),(10); +-- success, do update and insert only once +merge into t1 using t2 +on (t1.c1 = t2.c1) +when matched then update set c2 = 'i' +when not matched then insert values (9, 'i', '2023-09-24'); + +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 order by c1; + +reset behavior_compat_options; + +drop schema merge_into_selfmodified cascade; diff --git a/src/test/regress/sql/merge_into_updated.sql b/src/test/regress/sql/merge_into_updated.sql new file mode 100644 index 000000000..4d2b13af3 --- /dev/null +++ b/src/test/regress/sql/merge_into_updated.sql @@ -0,0 +1,118 @@ +create schema merge_into_updated; +set search_path = 'merge_into_updated'; + +create table t1 (c1 int, c2 text, c3 timestamp); +insert into t1 values (1, 'a', '2023-09-15'); + +create or replace function t1_tri_func() return trigger as +begin + new.c3 = '2023-09-16'; + return new; +end; +/ +create trigger t1_tri + before update on t1 + for each row + execute procedure t1_tri_func(); + +-- success, matched, do update +\parallel on 2 +begin + update t1 set c2 = 'b' where c1 = 1; + perform pg_sleep(3); +end; +/ +begin + perform pg_sleep(1); + merge into t1 using (select 1 c1) t2 + on (t1.c1 = t2.c1) + when matched then update set c2 = 'c' + when not matched then insert values (2, 'c', '2023-09-17'); +end; +/ +\parallel off +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 order by c1; + +-- success, matched, do update +\parallel on 2 +begin + update t1 set c2 = 'b' where c1 = 1; + perform pg_sleep(3); +end; +/ +begin + perform pg_sleep(1); + insert into t1 values (1, 'hello', '2023-09-17'); + delete from t1 where c2 = 'hello'; + merge into t1 using (select 1 c1) t2 + on (t1.c1 = t2.c1) + when matched then update set c2 = 'd' + when not matched then insert values (2, 'c', '2023-09-17'); +end; +/ +\parallel off +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 order by c1; + +-- success, concurrently update join condition, not matched, do insert +\parallel on 2 +begin + update t1 set c1 = 2 where c1 = 1; + perform pg_sleep(3); +end; +/ +begin + perform pg_sleep(1); + merge into t1 using (select 1 c1) t2 + on (t1.c1 = t2.c1) + when matched then update set c2 = 'd' + when not matched then insert values (3, 'd', '2023-09-18'); +end; +/ +\parallel off +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 order by c1; + +-- success, not matched, do insert +\parallel on 2 +begin + update t1 set c2 = 'b'; + perform pg_sleep(3); +end; +/ +begin + perform pg_sleep(1); + insert into t1 values (1, 'hello', '2023-09-17'); + delete from t1; + merge into t1 using (select 1 c1) t2 + on (t1.c1 = t2.c1) + when matched then update set c2 = 'c' + when not matched then insert values (2, 'c', '2023-09-17'); +end; +/ +\parallel off +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 order by c1; + +-- trigger update the join condition, not matched, do insert +create or replace function t1_tri_func() return trigger as +begin + new.c1 = 4; + return new; +end; +/ +\parallel on 2 +begin + update t1 set c2 = 'b'; + perform pg_sleep(3); +end; +/ +begin + perform pg_sleep(1); + merge into t1 using (select 2 c1) t2 + on (t1.c1 = t2.c1) + when matched then update set c2 = 'e' + when not matched then insert values (4, 'e', '2023-09-19'); +end; +/ +\parallel off +select c1, c2, to_char(c3, 'yyyy-mm-dd') from t1 order by c1; + +drop schema merge_into_updated cascade; From 2f907408b5f3ec8aa4e1201a3a951ff3e41de6f6 Mon Sep 17 00:00:00 2001 From: yanghao Date: Mon, 25 Sep 2023 15:08:28 +0800 Subject: [PATCH 286/304] change default value of enable_codegen to off --- src/common/backend/utils/misc/guc/guc_sql.cpp | 2 +- src/common/backend/utils/misc/postgresql_single.conf.sample | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/backend/utils/misc/guc/guc_sql.cpp b/src/common/backend/utils/misc/guc/guc_sql.cpp index 26115af26..dc3657a5a 100755 --- a/src/common/backend/utils/misc/guc/guc_sql.cpp +++ b/src/common/backend/utils/misc/guc/guc_sql.cpp @@ -581,7 +581,7 @@ static void InitSqlConfigureNamesBool() gettext_noop("Enable llvm for executor."), NULL}, &u_sess->attr.attr_sql.enable_codegen, - true, + false, NULL, NULL, NULL}, diff --git a/src/common/backend/utils/misc/postgresql_single.conf.sample b/src/common/backend/utils/misc/postgresql_single.conf.sample index 3e78fabea..d75e5483a 100644 --- a/src/common/backend/utils/misc/postgresql_single.conf.sample +++ b/src/common/backend/utils/misc/postgresql_single.conf.sample @@ -800,7 +800,7 @@ audit_enabled = on #------------------------------------------------------------------------------ # LLVM #------------------------------------------------------------------------------ -#enable_codegen = on # consider use LLVM optimization +#enable_codegen = off # consider use LLVM optimization #enable_codegen_print = off # dump the IR function #codegen_cost_threshold = 10000 # the threshold to allow use LLVM Optimization From 766a2b9a5458b65c5a24bebb9c4b5c50309d60a7 Mon Sep 17 00:00:00 2001 From: totaj Date: Mon, 25 Sep 2023 15:27:04 +0800 Subject: [PATCH 287/304] Fix load dolphin bug when upgrade in lite-mode. --- src/gausskernel/process/tcop/postgres.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/gausskernel/process/tcop/postgres.cpp b/src/gausskernel/process/tcop/postgres.cpp index 74e69a81c..2ccce0002 100755 --- a/src/gausskernel/process/tcop/postgres.cpp +++ b/src/gausskernel/process/tcop/postgres.cpp @@ -7640,7 +7640,13 @@ void RemoveTempNamespace() void LoadSqlPlugin() { if (u_sess->proc_cxt.MyDatabaseId != InvalidOid && DB_IS_CMPT(B_FORMAT) && IsFileExisted(DOLPHIN)) { - if (!u_sess->attr.attr_sql.dolphin && !u_sess->attr.attr_common.IsInplaceUpgrade) { + if (!u_sess->attr.attr_sql.dolphin && +#ifdef ENABLE_LITE_MODE + u_sess->attr.attr_common.upgrade_mode == 0 +#else + !u_sess->attr.attr_common.IsInplaceUpgrade +#endif + ) { Oid userId = GetUserId(); if (userId != INITIAL_USER_ID) { ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), From e14f9a701db873eb11f838296ddcfd3084b3026b Mon Sep 17 00:00:00 2001 From: openGaussDev Date: Mon, 25 Sep 2023 16:47:45 +0800 Subject: [PATCH 288/304] =?UTF-8?q?=20gs=5Frestore=E5=AF=BC=E5=85=A5utf8?= =?UTF-8?q?=E6=A0=BC=E5=BC=8F=E6=95=B0=E6=8D=AE=E5=88=B0gb18030-2022?= =?UTF-8?q?=E5=BA=93=E4=B8=AD=EF=BC=8C=E6=9F=A5=E8=AF=A2=E5=A4=B1=E8=B4=A5?= =?UTF-8?q?=E9=97=AE=E9=A2=98Offering:=20openGaussDevMore=20detail:=20gs?= =?UTF-8?q?=5Frestore=E5=AF=BC=E5=85=A5utf8=E6=A0=BC=E5=BC=8F=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E5=88=B0gb18030-2022=E5=BA=93=E4=B8=AD=EF=BC=8C?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E5=A4=B1=E8=B4=A5=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Match-id-d255d59f82fcade171a6204cbc79c3a1b5348128 --- src/gausskernel/optimizer/commands/copy.cpp | 25 ++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/gausskernel/optimizer/commands/copy.cpp b/src/gausskernel/optimizer/commands/copy.cpp index d95db1036..4a6cf6a5b 100644 --- a/src/gausskernel/optimizer/commands/copy.cpp +++ b/src/gausskernel/optimizer/commands/copy.cpp @@ -7418,6 +7418,8 @@ static int CopyReadAttributesTextT(CopyState cstate) int input_len; bool saw_non_ascii = false; proc_col_num++; + int byte_count = 0; + int pos = 0; /* Make sure there is enough space for the next value */ if (fieldno >= cstate->max_fields) { @@ -7471,8 +7473,29 @@ static int CopyReadAttributesTextT(CopyState cstate) found_delim = true; break; } + if (PG_GB18030 == GetDatabaseEncoding() || PG_GB18030_2022 == GetDatabaseEncoding()) { + if (pos == byte_count) { + unsigned char c1 = (unsigned char)(c); + if (c1 < (unsigned char)0x80) { + byte_count = 1; + } else if (c1 >= (unsigned char)0x81 && c1 <= (unsigned char)0xFE) { + char nextc = *cur_ptr; + unsigned char nextc1 = (unsigned char)(nextc); + if (nextc1 >= (unsigned char)0x30 && nextc1 <= (unsigned char)0x39) { + byte_count = 4; + } else { + byte_count = 2; + } + } else { + ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), + errmsg("invalid character encoding in gb18030 or gb18030_2022"))); + } + pos = 0; + } + pos++; + } - if (c == '\\' && !cstate->without_escaping) { + if (c == '\\' && !cstate->without_escaping && byte_count != 2) { if (cur_ptr >= line_end_ptr) { break; } From a8784c8ab43c0e7bd28505400ac5b440c08480d2 Mon Sep 17 00:00:00 2001 From: dongning12 Date: Mon, 25 Sep 2023 21:29:05 +0800 Subject: [PATCH 289/304] =?UTF-8?q?=E3=80=90=E8=B5=84=E6=BA=90=E6=B1=A0?= =?UTF-8?q?=E5=8C=96=E3=80=91=E5=90=8C=E6=AD=A5DMS=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/ddes/ddes_commit_id | 2 +- src/include/ddes/dms/dms_api.h | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/gausskernel/ddes/ddes_commit_id b/src/gausskernel/ddes/ddes_commit_id index fb48bf880..b3764787f 100644 --- a/src/gausskernel/ddes/ddes_commit_id +++ b/src/gausskernel/ddes/ddes_commit_id @@ -1,3 +1,3 @@ -dms_commit_id=a6c2fef3b8c08c11e4cee1adcacafabdf3b2f56b +dms_commit_id=f550952d5acd745d53f07bfac2edcd8c87e1e897 dss_commit_id=13bb5ba33820961455093f838977c54ba0867f28 cbb_commit_id=a7f55f6371988695578304bfbd3450f6f31c6630 \ No newline at end of file diff --git a/src/include/ddes/dms/dms_api.h b/src/include/ddes/dms/dms_api.h index 88cb535a7..74dfc19e1 100644 --- a/src/include/ddes/dms/dms_api.h +++ b/src/include/ddes/dms/dms_api.h @@ -685,6 +685,7 @@ typedef void (*dms_clean_ctrl_edp)(void *db_handle, dms_buf_ctrl_t *dms_ctrl); typedef char *(*dms_display_pageid)(char *display_buf, unsigned int count, char *pageid); typedef char *(*dms_display_xid)(char *display_buf, unsigned int count, char *xid); typedef char *(*dms_display_rowid)(char *display_buf, unsigned int count, char *rowid); +typedef int (*dms_check_session_invalid)(unsigned int sid); typedef int (*dms_drc_buf_res_rebuild)(void *db_handle); typedef int (*dms_drc_buf_res_rebuild_parallel)(void *db_handle, unsigned char thread_index, unsigned char thread_num); typedef int(*dms_ctl_rcy_clean_parallel_t)(void *db_handle, unsigned char thread_index, unsigned char thread_num); @@ -847,6 +848,7 @@ typedef struct st_dms_callback { dms_display_pageid display_pageid; dms_display_xid display_xid; dms_display_rowid display_rowid; + dms_check_session_invalid check_session_invalid; // for smon deadlock check dms_get_sid_by_rmid get_sid_by_rmid; @@ -883,6 +885,7 @@ typedef struct st_dms_callback { //for shared storage backup dms_set_inst_behavior set_inst_behavior; dms_db_prepare db_prepare; + dms_get_buf_info get_buf_info; } dms_callback_t; @@ -967,7 +970,7 @@ typedef enum st_dms_protocol_version { #define DMS_LOCAL_MINOR_VER_WEIGHT 1000 #define DMS_LOCAL_MAJOR_VERSION 0 #define DMS_LOCAL_MINOR_VERSION 0 -#define DMS_LOCAL_VERSION 94 +#define DMS_LOCAL_VERSION 95 #ifdef __cplusplus } From 6cfc687dac706473a543fb241300c08dcae26d90 Mon Sep 17 00:00:00 2001 From: j30049513 Date: Tue, 26 Sep 2023 11:10:01 +0800 Subject: [PATCH 290/304] =?UTF-8?q?fastcheck=E5=A2=9E=E5=8A=A0=E8=BF=9B?= =?UTF-8?q?=E5=BA=A6=E6=9D=A1=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/test/regress/pg_regress.cpp | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/test/regress/pg_regress.cpp b/src/test/regress/pg_regress.cpp index 34b0c7e26..7ede9947b 100644 --- a/src/test/regress/pg_regress.cpp +++ b/src/test/regress/pg_regress.cpp @@ -4712,6 +4712,24 @@ static int regrReloadAndParseLineBuffer(bool* pbBuffReloadReq, bool* pbHalfReadT return iRet; } +static int countTestLines(const char* filename){ + char command[1024]; + snprintf(command, sizeof(command), "grep -c '^test:' %s", filename); + + FILE *f = popen(command, "r"); + if(f == NULL){ + fprintf(stderr, "Failed to execute command.\n"); + return -1; + } + + char result[10]; + fgets(result, sizeof(result), f); + pclose(f); + + int count = atoi((result)); + return count; +} + /* * Run all the tests specified in one schedule file */ @@ -4734,6 +4752,8 @@ static void run_schedule(const char* schedule, test_function tfunc, diag_functio bool bIgnoreLineOnReload = false; bool isSystemTableDDL = false; bool isPlanAndProto = false; + int all_test_lines = 0; + int done_test_lines = 0; if (grayscale_upgrade != -1) { g_uiTotalBuf = 0; } @@ -4776,6 +4796,7 @@ static void run_schedule(const char* schedule, test_function tfunc, diag_functio /* Initializing the "total time taken by the test suite execution" */ g_dGroupTotalTime = 0; + all_test_lines = countTestLines(schedule); scf = fopen(schedule, "r"); if (!scf) { fprintf(stderr, _("%s: could not open file \"%s\" for reading: %s\n"), progname, schedule, gs_strerror(errno)); @@ -4950,7 +4971,7 @@ static void run_schedule(const char* schedule, test_function tfunc, diag_functio } else if (use_jdbc_client) { status(_("jdbc test %-24s .... "), tests[0]); } else { - status(_("test %-24s .... "), tests[0]); + status(_("test(%d/%d) %-24s .... "), ++done_test_lines, all_test_lines, tests[0]); } makeNestedDirectory(tests[0]); @@ -5079,7 +5100,7 @@ static void run_schedule(const char* schedule, test_function tfunc, diag_functio } else if (isPlanAndProto) { status(_("parallel group (%d plan_proto_tests): "), num_tests); } else { - status(_("parallel group (%d tests): "), num_tests); + status(_("parallel group (%d tests)(%d/%d): "), num_tests, ++done_test_lines, all_test_lines); } wait_for_tests(pids, statuses, tests, num_tests); From a66f189ad3131108a02ff9da42a45ad82ef74385 Mon Sep 17 00:00:00 2001 From: openGaussDev Date: Sun, 24 Sep 2023 15:53:44 +0800 Subject: [PATCH 291/304] Offering: openGaussDev More detail:fix RTO&Standby_Read params pro --- src/bin/gs_guc/cluster_guc.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bin/gs_guc/cluster_guc.conf b/src/bin/gs_guc/cluster_guc.conf index aeb8f93e6..af5aaea40 100755 --- a/src/bin/gs_guc/cluster_guc.conf +++ b/src/bin/gs_guc/cluster_guc.conf @@ -714,8 +714,8 @@ undo_zone_count|int|0,1048576|NULL|NULL| stream_cluster_run_mode|enum|cluster_primary,cluster_standby|NULL|NULL| xlog_file_size|int64|1048576,576460752303423487|B|The value must be an integer multiple of 16777216(16M)| xlog_file_path|string|0,0|NULL|NULL| -max_standby_base_page_size|int64|1073741824,576460752303423487|B|NULL| -max_standby_lsn_info_size|int64|1073741824,576460752303423487|B|NULL| +max_standby_base_page_size|int64|1048576,562949953421311|kB|NULL| +max_standby_lsn_info_size|int64|1048576,562949953421311|kB|NULL| plsql_show_all_error|bool|0,0|NULL|NULL| partition_page_estimation|bool|0,0|NULL|NULL| enable_auto_clean_unique_sql|bool|0,0|NULL|NULL| From 3303531ce9462ecdee64416a57cbcdebde20c5bb Mon Sep 17 00:00:00 2001 From: dongning12 Date: Tue, 26 Sep 2023 15:17:08 +0800 Subject: [PATCH 292/304] =?UTF-8?q?=E3=80=90=E8=B5=84=E6=BA=90=E6=B1=A0?= =?UTF-8?q?=E5=8C=96=E3=80=91SSWaitIOTimeout=E4=BF=AE=E5=A4=8D+DMS?= =?UTF-8?q?=E6=8E=A8=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/ddes/adapter/ss_dms_bufmgr.cpp | 2 ++ src/gausskernel/ddes/ddes_commit_id | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/gausskernel/ddes/adapter/ss_dms_bufmgr.cpp b/src/gausskernel/ddes/adapter/ss_dms_bufmgr.cpp index d9df56c34..2753f87b3 100644 --- a/src/gausskernel/ddes/adapter/ss_dms_bufmgr.cpp +++ b/src/gausskernel/ddes/adapter/ss_dms_bufmgr.cpp @@ -989,6 +989,8 @@ bool SSWaitIOTimeout(BufferDesc *buf) ret = SSLWLockAcquireTimeout(buf->io_in_progress_lock, LW_SHARED); if (ret) { LWLockRelease(buf->io_in_progress_lock); + } else { + break; } } diff --git a/src/gausskernel/ddes/ddes_commit_id b/src/gausskernel/ddes/ddes_commit_id index b3764787f..060aec9be 100644 --- a/src/gausskernel/ddes/ddes_commit_id +++ b/src/gausskernel/ddes/ddes_commit_id @@ -1,3 +1,3 @@ -dms_commit_id=f550952d5acd745d53f07bfac2edcd8c87e1e897 +dms_commit_id=499845395d8ad1f0fc7d5d9da9ee86a6b39cf889 dss_commit_id=13bb5ba33820961455093f838977c54ba0867f28 cbb_commit_id=a7f55f6371988695578304bfbd3450f6f31c6630 \ No newline at end of file From e8c51e1bab9e3fa85d9e1fca13f88221071295e7 Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Tue, 26 Sep 2023 19:40:26 +0800 Subject: [PATCH 293/304] =?UTF-8?q?=E8=B0=83=E6=95=B4=E6=A0=BC=E5=BC=8F?= =?UTF-8?q?=EF=BC=8C=E8=A7=A3=E5=86=B3=E7=BC=96=E8=AF=91=E5=91=8A=E8=AD=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/initdb/initdb.cpp | 12 ++++++------ src/bin/pg_ctl/pg_ctl.cpp | 2 +- src/common/backend/utils/misc/guc/guc_storage.cpp | 10 +++++----- .../backend/utils/misc/postgresql_single.conf.sample | 2 +- src/gausskernel/ddes/adapter/ss_reform_common.cpp | 4 ++-- src/include/knl/knl_guc/knl_instance_attr_storage.h | 2 +- src/include/replication/ss_cluster_replication.h | 2 +- src/test/regress/output/recovery_2pc_tools.source | 2 +- 8 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/bin/initdb/initdb.cpp b/src/bin/initdb/initdb.cpp index 676432793..12fb5d565 100644 --- a/src/bin/initdb/initdb.cpp +++ b/src/bin/initdb/initdb.cpp @@ -173,7 +173,7 @@ static char* xlog_dir = ""; static bool security = false; static char* dbcompatibility = ""; static char* new_xlog_file_path = ""; -static bool enable_ss_dorado = false; +static bool ss_enable_dorado = false; #ifdef PGXC /* Name of the PGXC node initialized */ @@ -1600,10 +1600,10 @@ static void setup_config(void) conflines = ss_addnodeparmater(conflines); } - if (enable_ss_dorado) { - nRet = strcpy_s(repltok, sizeof(repltok), "enable_ss_dorado = on"); + if (ss_enable_dorado) { + nRet = strcpy_s(repltok, sizeof(repltok), "ss_enable_dorado = on"); securec_check_c(nRet, "\0", "\0"); - conflines = replace_token(conflines, "#enable_ss_dorado = off", repltok); + conflines = replace_token(conflines, "#ss_enable_dorado = off", repltok); } nRet = sprintf_s(path, sizeof(path), "%s/postgresql.conf", pg_data); @@ -4305,7 +4305,7 @@ int main(int argc, char* argv[]) ss_nodedatainfo = xstrdup(optarg); break; case 19: - enable_ss_dorado = true; + ss_enable_dorado = true; printf(_("Enable ss dorado replication.\n")); break; #endif @@ -4932,7 +4932,7 @@ int main(int argc, char* argv[]) } if (enable_dss && ss_issharedstorage) { - ss_need_mkspecialdir = (enable_ss_dorado && !ss_check_specialdir(vgdata)); + ss_need_mkspecialdir = (ss_enable_dorado && !ss_check_specialdir(vgdata)); ss_mkdirdir(ss_nodeid, pg_data, vgdata, vglog, ss_need_mkclusterdir, ss_need_mkspecialdir); } else { /* Create required subdirectories */ diff --git a/src/bin/pg_ctl/pg_ctl.cpp b/src/bin/pg_ctl/pg_ctl.cpp index 2d0581118..e4edb88b5 100755 --- a/src/bin/pg_ctl/pg_ctl.cpp +++ b/src/bin/pg_ctl/pg_ctl.cpp @@ -4918,7 +4918,7 @@ static bool DoBuildCheck(uint32 term) buildSuccess = build_check_main(term); if (!buildSuccess) { - pg_log(PG_WARNING, _("Build check result : full build\n"), BuildModeToString(build_mode), pg_data); + pg_log(PG_WARNING, _("Build check result : full build\n")); pg_log(PG_WARNING, _("%s failed(%s).\n"), BuildModeToString(build_mode), pg_data); } else { pg_log(PG_WARNING, _("%s completed(%s).\n"), BuildModeToString(build_mode), pg_data); diff --git a/src/common/backend/utils/misc/guc/guc_storage.cpp b/src/common/backend/utils/misc/guc/guc_storage.cpp index 6e1ec9c8a..d9ecd7dc9 100755 --- a/src/common/backend/utils/misc/guc/guc_storage.cpp +++ b/src/common/backend/utils/misc/guc/guc_storage.cpp @@ -1136,14 +1136,14 @@ static void InitStorageConfigureNamesBool() NULL, NULL, NULL}, - {{"enable_ss_dorado", + {{"ss_enable_dorado", PGC_POSTMASTER, NODE_SINGLENODE, WAL, gettext_noop("Use to enabel dorado replication in share storage mode."), NULL, GUC_SUPERUSER_ONLY}, - &g_instance.attr.attr_storage.enable_ss_dorado, + &g_instance.attr.attr_storage.ss_enable_dorado, false, check_ss_cluster_replication_control_para, NULL, @@ -6117,10 +6117,10 @@ static bool check_normal_cluster_replication_config_para(char** newval, void** e return true; } - if (g_instance.attr.attr_storage.enable_ss_dorado) { + if (g_instance.attr.attr_storage.ss_enable_dorado) { ereport(ERROR, (errmsg("Do not allow both enable normal cluster replication " - "and ss cluster repliction with \"enable_ss_dorado\" = %d", \ - g_instance.attr.attr_storage.enable_ss_dorado))); + "and ss cluster repliction with \"ss_enable_dorado\" = %d", \ + g_instance.attr.attr_storage.ss_enable_dorado))); return false; } diff --git a/src/common/backend/utils/misc/postgresql_single.conf.sample b/src/common/backend/utils/misc/postgresql_single.conf.sample index d75e5483a..ca0797382 100644 --- a/src/common/backend/utils/misc/postgresql_single.conf.sample +++ b/src/common/backend/utils/misc/postgresql_single.conf.sample @@ -851,5 +851,5 @@ job_queue_processes = 10 # Number of concurrent jobs, optional: [0..1000] #ss_parallel_thread_count = 16 #ss_enable_ondemand_recovery = off #ss_ondemand_recovery_mem_size = 4GB # min: 1GB, max: 100GB -#enable_ss_dorado = off +#ss_enable_dorado = off #enable_segment = off diff --git a/src/gausskernel/ddes/adapter/ss_reform_common.cpp b/src/gausskernel/ddes/adapter/ss_reform_common.cpp index 1871fceaf..844aa32c3 100644 --- a/src/gausskernel/ddes/adapter/ss_reform_common.cpp +++ b/src/gausskernel/ddes/adapter/ss_reform_common.cpp @@ -233,8 +233,8 @@ void SSDoradoGetInstidList() g_instance.attr.attr_storage.dss_attr.ss_dss_vg_name))); } - int len = strlen("pg_xlog"); - int index = 0; + uint8_t len = strlen("pg_xlog"); + uint8_t index = 0; while ((entry = readdir(dssdir)) != NULL) { if (strncmp(entry->d_name, "pg_xlog", len) == 0) { if (strlen(entry->d_name) > len) { diff --git a/src/include/knl/knl_guc/knl_instance_attr_storage.h b/src/include/knl/knl_guc/knl_instance_attr_storage.h index f32e80dde..2077bb691 100755 --- a/src/include/knl/knl_guc/knl_instance_attr_storage.h +++ b/src/include/knl/knl_guc/knl_instance_attr_storage.h @@ -224,7 +224,7 @@ typedef struct knl_instance_attr_storage { bool enable_batch_dispatch; int parallel_recovery_timeout; int parallel_recovery_batch; - bool enable_ss_dorado; + bool ss_enable_dorado; } knl_instance_attr_storage; #endif /* SRC_INCLUDE_KNL_KNL_INSTANCE_ATTR_STORAGE_H_ */ diff --git a/src/include/replication/ss_cluster_replication.h b/src/include/replication/ss_cluster_replication.h index 17b81ca54..eebffcff2 100644 --- a/src/include/replication/ss_cluster_replication.h +++ b/src/include/replication/ss_cluster_replication.h @@ -33,7 +33,7 @@ const uint32 SS_DORADO_CTL_INFO_SIZE = 512; #define SS_REPLICATION_DORADO_CLUSTER \ - (ENABLE_DSS && g_instance.attr.attr_storage.enable_ss_dorado) + (ENABLE_DSS && g_instance.attr.attr_storage.ss_enable_dorado) /* Primary Cluster in SS replication */ #define SS_REPLICATION_PRIMARY_CLUSTER \ diff --git a/src/test/regress/output/recovery_2pc_tools.source b/src/test/regress/output/recovery_2pc_tools.source index 0cd8b20ed..cc81133c9 100644 --- a/src/test/regress/output/recovery_2pc_tools.source +++ b/src/test/regress/output/recovery_2pc_tools.source @@ -326,7 +326,7 @@ select name,vartype,unit,min_val,max_val from pg_settings where name <> 'qunit_c enable_sonic_optspill | bool | | | enable_sort | bool | | | enable_sortgroup_agg | bool | | | - enable_ss_dorado | bool | | | + ss_enable_dorado | bool | | | enable_startwith_debug | bool | | | enable_stmt_track | bool | | | enable_stream_replication | bool | | | From bc4e1013bbae19226ec67afa31beb94dfbc3f5a9 Mon Sep 17 00:00:00 2001 From: laishenghao Date: Tue, 26 Sep 2023 19:51:10 +0800 Subject: [PATCH 294/304] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E5=A4=9A=E8=A1=8C?= =?UTF-8?q?=E6=8F=92=E5=85=A5=E5=8C=85=E5=90=AB=E5=8F=B3=E5=80=BC=E5=BC=95?= =?UTF-8?q?=E7=94=A8=E6=97=B6=E5=AD=98=E5=9C=A8=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/backend/parser/analyze.cpp | 15 +++--- src/gausskernel/runtime/executor/execQual.cpp | 4 +- .../runtime/executor/execUtils.cpp | 7 ++- .../runtime/executor/nodeValuesscan.cpp | 23 +++++----- src/include/executor/executor.h | 3 +- .../regress/input/insert_right_ref.source | 18 ++++++++ .../regress/output/insert_right_ref.source | 46 +++++++++++++++++++ 7 files changed, 90 insertions(+), 26 deletions(-) diff --git a/src/common/backend/parser/analyze.cpp b/src/common/backend/parser/analyze.cpp index b00bca043..9fa5ca49b 100644 --- a/src/common/backend/parser/analyze.cpp +++ b/src/common/backend/parser/analyze.cpp @@ -1594,17 +1594,16 @@ static void CheckUnsupportInsertSelectClause(Query* query) } -static void SetInsertAttrnoState(ParseState* pstate, List* attrnos) +static void SetInsertAttrnoState(ParseState* pstate, List* attrnos, int exprLen) { RightRefState* rstate = pstate->rightRefState; Relation relation = (Relation)linitial(pstate->p_target_relation); rstate->colCnt = RelationGetNumberOfAttributes(relation); - int len = list_length(attrnos); - rstate->explicitAttrLen = len; - rstate->explicitAttrNos = (int*)palloc0(sizeof(int) * len); + rstate->explicitAttrLen = exprLen; + rstate->explicitAttrNos = (int*)palloc0(sizeof(int) * exprLen); ListCell* attr = list_head(attrnos); - for (int i = 0; i < len; ++i) { + for (int i = 0; i < exprLen; ++i) { rstate->explicitAttrNos[i] = lfirst_int(attr); attr = lnext(attr); } @@ -1854,8 +1853,6 @@ static Query* transformInsertStmt(ParseState* pstate, InsertStmt* stmt) /* Validate stmt->cols list, or build default list if no list given */ icolumns = checkInsertTargets(pstate, stmt->cols, &attrnos); - - SetInsertAttrnoState(pstate, attrnos); AssertEreport(list_length(icolumns) == list_length(attrnos), MOD_OPT, "list length inconsistent"); @@ -2179,6 +2176,10 @@ static Query* transformInsertStmt(ParseState* pstate, InsertStmt* stmt) exprList = transformInsertRow(pstate, exprList, stmt->cols, icolumns, attrnos); } + if (rightRefState->isSupported) { + SetInsertAttrnoState(pstate, attrnos, list_length(exprList)); + } + /* * Generate query's target list using the computed list of expressions. * Also, mark all the target columns as needing insert permissions. diff --git a/src/gausskernel/runtime/executor/execQual.cpp b/src/gausskernel/runtime/executor/execQual.cpp index a5e822161..25edc6283 100644 --- a/src/gausskernel/runtime/executor/execQual.cpp +++ b/src/gausskernel/runtime/executor/execQual.cpp @@ -632,7 +632,7 @@ static Datum ExecEvalScalarVar(ExprState* exprstate, ExprContext* econtext, bool RightRefState* refState = econtext->rightRefState; int index = attnum - 1; if (refState && refState->values && - (IS_ENABLE_INSERT_RIGHT_REF(refState) || + ((slot == nullptr && IS_ENABLE_INSERT_RIGHT_REF(refState)) || (IS_ENABLE_UPSERT_RIGHT_REF(refState) && refState->hasExecs[index] && index < refState->colCnt))) { *isNull = refState->isNulls[index]; return refState->values[index]; @@ -6872,7 +6872,7 @@ static bool ExecTargetList(List* targetlist, ExprContext* econtext, Datum* value SortTargetListAsArray(refState, targetlist, targetArr); - InitOutputValues(refState, targetArr, values, isnull, targetCount, hasExecs); + InitOutputValues(refState, values, isnull, hasExecs); /* * evaluate all the expressions in the target list diff --git a/src/gausskernel/runtime/executor/execUtils.cpp b/src/gausskernel/runtime/executor/execUtils.cpp index 06d632b61..a07903b34 100644 --- a/src/gausskernel/runtime/executor/execUtils.cpp +++ b/src/gausskernel/runtime/executor/execUtils.cpp @@ -2825,8 +2825,7 @@ Tuple ReplaceTupleNullCol(TupleDesc tupleDesc, TupleTableSlot *slot) } -void InitOutputValues(RightRefState* refState, GenericExprState* targetArr[], - Datum* values, bool* isnull, int targetCount, bool* hasExecs) +void InitOutputValues(RightRefState* refState, Datum* values, bool* isnull, bool* hasExecs) { if (!IS_ENABLE_RIGHT_REF(refState)) { return; @@ -2835,13 +2834,13 @@ void InitOutputValues(RightRefState* refState, GenericExprState* targetArr[], refState->values = values; refState->isNulls = isnull; refState->hasExecs = hasExecs; - int colCnt = refState->colCnt; + const int colCnt = refState->colCnt; for (int i = 0; i < colCnt; ++i) { hasExecs[i] = false; } if (IS_ENABLE_INSERT_RIGHT_REF(refState)) { - for (int i = 0; i < targetCount; ++i) { + for (int i = 0; i < colCnt; ++i) { Const* con = refState->constValues[i]; if (con) { values[i] = con->constvalue; diff --git a/src/gausskernel/runtime/executor/nodeValuesscan.cpp b/src/gausskernel/runtime/executor/nodeValuesscan.cpp index 210b66a17..606836940 100644 --- a/src/gausskernel/runtime/executor/nodeValuesscan.cpp +++ b/src/gausskernel/runtime/executor/nodeValuesscan.cpp @@ -125,30 +125,31 @@ static TupleTableSlot* ValuesNext(ValuesScanState* node) RightRefState* refState = econtext->rightRefState; int targetCount = list_length(expr_state_list); - GenericExprState* targetArr[targetCount]; int colCnt = (IS_ENABLE_RIGHT_REF(refState) && refState->colCnt > 0) ? refState->colCnt : 1; bool hasExecs[colCnt]; - - SortTargetListAsArray(refState, expr_state_list, targetArr); - - InitOutputValues(refState, targetArr, values, is_null, targetCount, hasExecs); + Datum rightRefValues[colCnt]; + bool rightRefIsNulls[colCnt]; + InitOutputValues(refState, rightRefValues, rightRefIsNulls, hasExecs); resind = 0; foreach (lc, expr_state_list) { ExprState* exprState = (ExprState*)lfirst(lc); values[resind] = ExecEvalExpr(exprState, econtext, &is_null[resind]); - if (IS_ENABLE_RIGHT_REF(refState) && resind < refState->colCnt) { - hasExecs[resind] = true; + if (unlikely(IS_ENABLE_INSERT_RIGHT_REF(refState) && resind < refState->explicitAttrLen)) { + int idx = refState->explicitAttrNos[resind] - 1; + hasExecs[idx] = true; + rightRefValues[idx] = values[resind]; + rightRefIsNulls[idx] = is_null[resind]; } resind++; } - if (IS_ENABLE_RIGHT_REF(econtext->rightRefState)) { - econtext->rightRefState->values = nullptr; - econtext->rightRefState->isNulls = nullptr; - econtext->rightRefState->hasExecs = nullptr; + if (unlikely(IS_ENABLE_RIGHT_REF(refState))) { + refState->values = nullptr; + refState->isNulls = nullptr; + refState->hasExecs = nullptr; } MemoryContextSwitchTo(old_context); diff --git a/src/include/executor/executor.h b/src/include/executor/executor.h index 49358b835..bfcb01803 100755 --- a/src/include/executor/executor.h +++ b/src/include/executor/executor.h @@ -507,8 +507,7 @@ extern TupleDesc ExecTypeFromTL(List *targetList, bool hasoid, bool markdropped extern TupleDesc ExecCleanTypeFromTL(List *targetList, bool hasoid, const TableAmRoutine *tam_ops = TableAmHeap); extern TupleDesc ExecTypeFromExprList(List *exprList, List *namesList, const TableAmRoutine *tam_ops = TableAmHeap); extern void UpdateChangedParamSet(PlanState *node, Bitmapset *newchg); -extern void InitOutputValues(RightRefState *refState, GenericExprState *targetArr[], Datum *values, bool *isnull, - int targetCount, bool *hasExecs); +extern void InitOutputValues(RightRefState* refState, Datum* values, bool* isnull, bool* hasExecs); extern void SortTargetListAsArray(RightRefState *refState, List *targetList, GenericExprState *targetArr[]); typedef struct TupOutputState { diff --git a/src/test/regress/input/insert_right_ref.source b/src/test/regress/input/insert_right_ref.source index da39b5356..8c6101202 100644 --- a/src/test/regress/input/insert_right_ref.source +++ b/src/test/regress/input/insert_right_ref.source @@ -450,6 +450,24 @@ CREATE TABLE t2 ( INSERT INTO t2 VALUES (6, 6) ON DUPLICATE KEY UPDATE t2.col1 = t2.col2 + 1; INSERT INTO t2 VALUES (6, 6) ON DUPLICATE KEY UPDATE t2.col1 = extract(century from col4) * 100 + extract(isodow from col4); +-- multi values case1 +create table t_multi_values(a int not null primary key, b char(10)); +insert into t_multi_values values (1); +insert into t_multi_values values (a+2); +insert into t_multi_values values (a+5),(a+6); +insert into t_multi_values values (a+7, b),(a+8, concat(b, ' not display')), (a+9 + a * 3, b),(a + 10 + a * 3, 'display'); +select * from t_multi_values order by a; + +-- multi values case2 +create table t_multi_values2(f1 int primary key, f2 int, f3 int); +insert into t_multi_values2(f1, f3, f2) VALUES(1, f1 + 2, f3 + 3),(2, f1 + 2, f3 + 3),(3, f1 + 2, f3 + 3),(4, f1 + 2, f3 + 3); +select * from t_multi_values2 order by f1; + +-- multi values case3 +create table t_multi_values3(f1 int primary key, f2 int, f3 int); +insert into t_multi_values3(f1, f3) VALUES(1, f1 + 2),(2, f1 + 2),(3, f1 + 2),(4, f1 + 2); +select * from t_multi_values3 order by f1; + -- jdbc case DROP USER IF EXISTS rightref CASCADE; CREATE USER rightref WITH PASSWORD 'rightref@123'; diff --git a/src/test/regress/output/insert_right_ref.source b/src/test/regress/output/insert_right_ref.source index 4e05f4ee5..c9d919864 100644 --- a/src/test/regress/output/insert_right_ref.source +++ b/src/test/regress/output/insert_right_ref.source @@ -625,6 +625,52 @@ NOTICE: CREATE TABLE will create implicit sequence "t2_col5_seq" for serial col NOTICE: CREATE TABLE / PRIMARY KEY will create implicit index "t2_pkey" for table "t2" INSERT INTO t2 VALUES (6, 6) ON DUPLICATE KEY UPDATE t2.col1 = t2.col2 + 1; INSERT INTO t2 VALUES (6, 6) ON DUPLICATE KEY UPDATE t2.col1 = extract(century from col4) * 100 + extract(isodow from col4); +-- multi values case1 +create table t_multi_values(a int not null primary key, b char(10)); +NOTICE: CREATE TABLE / PRIMARY KEY will create implicit index "t_multi_values_pkey" for table "t_multi_values" +insert into t_multi_values values (1); +insert into t_multi_values values (a+2); +insert into t_multi_values values (a+5),(a+6); +insert into t_multi_values values (a+7, b),(a+8, concat(b, ' not display')), (a+9 + a * 3, b),(a + 10 + a * 3, 'display'); +select * from t_multi_values order by a; + a | b +----+------------ + 1 | + 2 | + 5 | + 6 | + 7 | + 8 | + 9 | + 10 | display +(8 rows) + +-- multi values case2 +create table t_multi_values2(f1 int primary key, f2 int, f3 int); +NOTICE: CREATE TABLE / PRIMARY KEY will create implicit index "t_multi_values2_pkey" for table "t_multi_values2" +insert into t_multi_values2(f1, f3, f2) VALUES(1, f1 + 2, f3 + 3),(2, f1 + 2, f3 + 3),(3, f1 + 2, f3 + 3),(4, f1 + 2, f3 + 3); +select * from t_multi_values2 order by f1; + f1 | f2 | f3 +----+----+---- + 1 | 6 | 3 + 2 | 7 | 4 + 3 | 8 | 5 + 4 | 9 | 6 +(4 rows) + +-- multi values case3 +create table t_multi_values3(f1 int primary key, f2 int, f3 int); +NOTICE: CREATE TABLE / PRIMARY KEY will create implicit index "t_multi_values3_pkey" for table "t_multi_values3" +insert into t_multi_values3(f1, f3) VALUES(1, f1 + 2),(2, f1 + 2),(3, f1 + 2),(4, f1 + 2); +select * from t_multi_values3 order by f1; + f1 | f2 | f3 +----+----+---- + 1 | | 3 + 2 | | 4 + 3 | | 5 + 4 | | 6 +(4 rows) + -- jdbc case DROP USER IF EXISTS rightref CASCADE; NOTICE: role "rightref" does not exist, skipping From 7293396463ab61579dcba8413dbc37da471a190a Mon Sep 17 00:00:00 2001 From: dongning12 Date: Tue, 26 Sep 2023 21:52:05 +0800 Subject: [PATCH 295/304] =?UTF-8?q?=E3=80=90=E8=B5=84=E6=BA=90=E6=B1=A0?= =?UTF-8?q?=E5=8C=96=E3=80=91=E5=90=8C=E6=AD=A5dms=E7=82=B9=20+=20DSS=20+?= =?UTF-8?q?=20CBB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/ddes/ddes_commit_id | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/gausskernel/ddes/ddes_commit_id b/src/gausskernel/ddes/ddes_commit_id index 060aec9be..b01145127 100644 --- a/src/gausskernel/ddes/ddes_commit_id +++ b/src/gausskernel/ddes/ddes_commit_id @@ -1,3 +1,3 @@ -dms_commit_id=499845395d8ad1f0fc7d5d9da9ee86a6b39cf889 -dss_commit_id=13bb5ba33820961455093f838977c54ba0867f28 -cbb_commit_id=a7f55f6371988695578304bfbd3450f6f31c6630 \ No newline at end of file +dms_commit_id=6d6bb9bebf376b5a2570c2f89b1db7eef1361851 +dss_commit_id=4d9fe8081427520e8f4f63b76d510f05e3448a77 +cbb_commit_id=7cd88a1dcc256df30f3ef84835cfc2e8d0573364 \ No newline at end of file From 563971b4c27e9574e395f309f0b04fc0e5ab09f6 Mon Sep 17 00:00:00 2001 From: movead Date: Wed, 27 Sep 2023 09:54:07 +0800 Subject: [PATCH 296/304] for parallel redo, change txn check lsn method from finish lsn to tying lsn --- .../access/transam/parallel_recovery/dispatcher.cpp | 4 ++-- .../access/transam/parallel_recovery/page_redo.cpp | 7 +++++++ .../access/transam/parallel_recovery/txn_redo.cpp | 10 +++++++--- src/include/access/parallel_recovery/dispatcher.h | 2 +- src/include/access/parallel_recovery/page_redo.h | 2 ++ 5 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/gausskernel/storage/access/transam/parallel_recovery/dispatcher.cpp b/src/gausskernel/storage/access/transam/parallel_recovery/dispatcher.cpp index 8c1ae4548..a34c36918 100755 --- a/src/gausskernel/storage/access/transam/parallel_recovery/dispatcher.cpp +++ b/src/gausskernel/storage/access/transam/parallel_recovery/dispatcher.cpp @@ -1935,13 +1935,13 @@ void GetReplayedRecPtrFromUndoWorkers(XLogRecPtr *readPtr, XLogRecPtr *endPtr) *endPtr = minEnd; } -void GetReplayedRecPtrFromWorkers(XLogRecPtr *endPtr) +void GetReplayingRecPtrFromWorkers(XLogRecPtr *endPtr) { XLogRecPtr minEnd = MAX_XLOG_REC_PTR; for (uint32 i = 0; i < g_dispatcher->pageWorkerCount; i++) { if (!RedoWorkerIsIdle(g_dispatcher->pageWorkers[i])) { - XLogRecPtr end = GetCompletedRecPtr(g_dispatcher->pageWorkers[i]); + XLogRecPtr end = GetReplyingRecPtr(g_dispatcher->pageWorkers[i]); if (XLByteLT(end, minEnd)) { minEnd = end; } diff --git a/src/gausskernel/storage/access/transam/parallel_recovery/page_redo.cpp b/src/gausskernel/storage/access/transam/parallel_recovery/page_redo.cpp index a7981319c..5a46bb1b8 100755 --- a/src/gausskernel/storage/access/transam/parallel_recovery/page_redo.cpp +++ b/src/gausskernel/storage/access/transam/parallel_recovery/page_redo.cpp @@ -411,6 +411,7 @@ void ApplyProcHead(RedoItem *head) while (head != NULL) { RedoItem *cur = head; g_redoWorker->current_item = &cur->record; + pg_atomic_write_u64((volatile uint64*)&g_redoWorker->curReplayingReadRecPtr, cur->record.ReadRecPtr); head = head->nextByWorker[g_redoWorker->id + 1]; ApplyAndFreeRedoItem(cur); } @@ -1022,6 +1023,12 @@ XLogRecPtr GetCompletedRecPtr(PageRedoWorker *worker) return pg_atomic_read_u64(&worker->lastReplayedEndRecPtr); } +XLogRecPtr GetReplyingRecPtr(PageRedoWorker *worker) +{ + pg_read_barrier(); + return pg_atomic_read_u64(&worker->curReplayingReadRecPtr); +} + /* automic write for lastReplayedReadRecPtr and lastReplayedEndRecPtr */ void SetCompletedReadEndPtr(PageRedoWorker *worker, XLogRecPtr readPtr, XLogRecPtr endPtr) { diff --git a/src/gausskernel/storage/access/transam/parallel_recovery/txn_redo.cpp b/src/gausskernel/storage/access/transam/parallel_recovery/txn_redo.cpp index 0079b4a61..077a79491 100644 --- a/src/gausskernel/storage/access/transam/parallel_recovery/txn_redo.cpp +++ b/src/gausskernel/storage/access/transam/parallel_recovery/txn_redo.cpp @@ -272,33 +272,36 @@ void ApplyReadyTxnLogRecords(TxnRedoWorker *worker, bool forceAll) while (item != NULL) { XLogReaderState *record = &item->record; XLogRecPtr lrEnd; + XLogRecPtr curRead; pg_atomic_write_u64(&worker->txn_trying_lsn, record->EndRecPtr); if (forceAll) { GetRedoStartTime(t_thrd.xlog_cxt.timeCost[TIME_COST_STEP_6]); XLogRecPtr lrRead; /* lastReplayedReadPtr */ GetReplayedRecPtrFromWorkers(&lrRead, &lrEnd); + GetReplayingRecPtrFromWorkers(&curRead); /* we need to get lastCompletedPageLSN as soon as possible,so */ /* we can not sleep here. */ XLogRecPtr oldReplayedPageLSN = InvalidXLogRecPtr; - while (XLByteLT(lrEnd, record->EndRecPtr)) { + while (XLByteLT(curRead, record->EndRecPtr)) { /* update lastreplaylsn */ if (!XLByteEQ(oldReplayedPageLSN, lrEnd)) { SetXLogReplayRecPtr(lrRead, lrEnd); oldReplayedPageLSN = lrEnd; } GetReplayedRecPtrFromWorkers(&lrRead, &lrEnd); + GetReplayingRecPtrFromWorkers(&curRead); RedoInterruptCallBack(); } CountRedoTime(t_thrd.xlog_cxt.timeCost[TIME_COST_STEP_6]); } - GetReplayedRecPtrFromWorkers(&lrEnd); + GetReplayingRecPtrFromWorkers(&curRead); /* * Make sure we can replay this record. This check is necessary * on the master and on the hot backup after it reaches consistency. */ - if (XLByteLE(record->EndRecPtr, lrEnd)) { + if (XLByteLE(record->EndRecPtr, curRead)) { item = ProcTxnItem(item); } else { break; @@ -313,6 +316,7 @@ void ApplyReadyTxnLogRecords(TxnRedoWorker *worker, bool forceAll) XLogRecPtr oldReplayedPageLSN = InvalidXLogRecPtr; XLogRecPtr lrRead; XLogRecPtr lrEnd; + XLogRecPtr curRead; do { GetReplayedRecPtrFromWorkers(&lrRead, &lrEnd); if (XLByteLT(g_dispatcher->dispatchEndRecPtr, lrEnd)) { diff --git a/src/include/access/parallel_recovery/dispatcher.h b/src/include/access/parallel_recovery/dispatcher.h index 047c36002..86c173e4b 100644 --- a/src/include/access/parallel_recovery/dispatcher.h +++ b/src/include/access/parallel_recovery/dispatcher.h @@ -123,7 +123,7 @@ uint32 GetWorkerId(const RelFileNode& node, BlockNumber block, ForkNumber forkNu XLogReaderState* NewReaderState(XLogReaderState* readerState, bool bCopyState = false); void FreeAllocatedRedoItem(); void GetReplayedRecPtrFromWorkers(XLogRecPtr *readPtr, XLogRecPtr *endPtr); -void GetReplayedRecPtrFromWorkers(XLogRecPtr *endPtr); +void GetReplayingRecPtrFromWorkers(XLogRecPtr *endPtr); void GetReplayedRecPtrFromUndoWorkers(XLogRecPtr *readPtr, XLogRecPtr *endPtr); List* CheckImcompleteAction(List* imcompleteActionList); void SetPageWorkStateByThreadId(uint32 threadState); diff --git a/src/include/access/parallel_recovery/page_redo.h b/src/include/access/parallel_recovery/page_redo.h index 22dbfa51a..b1a4aadfb 100644 --- a/src/include/access/parallel_recovery/page_redo.h +++ b/src/include/access/parallel_recovery/page_redo.h @@ -72,6 +72,7 @@ struct PageRedoWorker { */ XLogRecPtr lastReplayedReadRecPtr; XLogRecPtr lastReplayedEndRecPtr; + XLogRecPtr curReplayingReadRecPtr; #if (!defined __x86_64__) && (!defined __aarch64__) /* protects lastReplayedReadRecPtr and lastReplayedEndRecPtr */ slock_t ptrLck; @@ -223,6 +224,7 @@ bool ProcessPendingPageRedoItems(PageRedoWorker* worker); /* Run-time worker states. */ uint64 GetCompletedRecPtr(PageRedoWorker* worker); +XLogRecPtr GetReplyingRecPtr(PageRedoWorker *worker); bool IsRecoveryRestartPointSafe(PageRedoWorker* worker, XLogRecPtr restartPoint); void SetWorkerRestartPoint(PageRedoWorker* worker, XLogRecPtr restartPoint); From d4e422e3ae1829103e794884ea928e916e0a785b Mon Sep 17 00:00:00 2001 From: dongning12 Date: Wed, 27 Sep 2023 15:16:52 +0800 Subject: [PATCH 297/304] =?UTF-8?q?=E3=80=90=E8=B5=84=E6=BA=90=E6=B1=A0?= =?UTF-8?q?=E5=8C=96=E3=80=91=E6=8E=A8CBB=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/ddes/ddes_commit_id | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gausskernel/ddes/ddes_commit_id b/src/gausskernel/ddes/ddes_commit_id index b01145127..b60c6ea19 100644 --- a/src/gausskernel/ddes/ddes_commit_id +++ b/src/gausskernel/ddes/ddes_commit_id @@ -1,3 +1,3 @@ dms_commit_id=6d6bb9bebf376b5a2570c2f89b1db7eef1361851 dss_commit_id=4d9fe8081427520e8f4f63b76d510f05e3448a77 -cbb_commit_id=7cd88a1dcc256df30f3ef84835cfc2e8d0573364 \ No newline at end of file +cbb_commit_id=d5e6d8009f4270e644d7c921f673979120422a54 \ No newline at end of file From 49cb438b8fabe8f6da3c6ca550fa5959f47b315c Mon Sep 17 00:00:00 2001 From: movead Date: Wed, 27 Sep 2023 15:56:28 +0800 Subject: [PATCH 298/304] in GetReplyingRecPtr pick bigger between curReplayingReadRecPtr and lastReplayedEndRecPtr; --- .../access/transam/parallel_recovery/page_redo.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/gausskernel/storage/access/transam/parallel_recovery/page_redo.cpp b/src/gausskernel/storage/access/transam/parallel_recovery/page_redo.cpp index 5a46bb1b8..52ac55e95 100755 --- a/src/gausskernel/storage/access/transam/parallel_recovery/page_redo.cpp +++ b/src/gausskernel/storage/access/transam/parallel_recovery/page_redo.cpp @@ -1025,8 +1025,15 @@ XLogRecPtr GetCompletedRecPtr(PageRedoWorker *worker) XLogRecPtr GetReplyingRecPtr(PageRedoWorker *worker) { + XLogRecPtr curReplayingReadRecPtr; + XLogRecPtr lastReplayedEndRecPtr; + XLogRecPtr result; pg_read_barrier(); - return pg_atomic_read_u64(&worker->curReplayingReadRecPtr); + + curReplayingReadRecPtr = pg_atomic_read_u64(&worker->curReplayingReadRecPtr); + lastReplayedEndRecPtr = pg_atomic_read_u64(&worker->lastReplayedEndRecPtr); + + return lastReplayedEndRecPtr > curReplayingReadRecPtr ? lastReplayedEndRecPtr : curReplayingReadRecPtr; } /* automic write for lastReplayedReadRecPtr and lastReplayedEndRecPtr */ From df2d37067dbb0a4b638f0bf3cf047978500473c2 Mon Sep 17 00:00:00 2001 From: yanghao Date: Wed, 27 Sep 2023 20:20:25 +0800 Subject: [PATCH 299/304] fix problem of rto standby read --- .../storage/access/redo/standby_read/standby_read_interface.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp b/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp index 8e307c14f..5c5c0588f 100644 --- a/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp @@ -935,7 +935,7 @@ Buffer standby_read_buf_new( UnlockReleaseBuffer(block_info_buf); return read_buf; } - UnlockReleaseBuffer(read_buf); + ReleaseBuffer(read_buf); extreme_rto::RedoItemTag redo_item_tag; INIT_REDO_ITEM_TAG(redo_item_tag, buf_tag.rnode, buf_tag.forkNum, buf_tag.blockNum); From cee74b1867721f4860b85bcf692b6149afaa1203 Mon Sep 17 00:00:00 2001 From: yanghao Date: Wed, 27 Sep 2023 20:26:28 +0800 Subject: [PATCH 300/304] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dcopy=E5=9C=A8?= =?UTF-8?q?=E8=A1=8C=E6=9C=AB=E5=AD=98=E5=9C=A8=E8=BD=AC=E4=B9=89=E5=AD=97?= =?UTF-8?q?=E7=AC=A6=E7=9A=84=E6=8A=A5=E9=94=99=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/gausskernel/optimizer/commands/copy.cpp | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/gausskernel/optimizer/commands/copy.cpp b/src/gausskernel/optimizer/commands/copy.cpp index 4a6cf6a5b..ff62e40fc 100644 --- a/src/gausskernel/optimizer/commands/copy.cpp +++ b/src/gausskernel/optimizer/commands/copy.cpp @@ -6984,6 +6984,24 @@ static bool CopyReadLineTextTemplate(CopyState cstate) if (raw_buf_ptr < copy_buf_len) { sec = copy_raw_buf[raw_buf_ptr]; } + if (IS_TEXT(cstate) && (cstate->copy_dest == COPY_NEW_FE) && !cstate->is_load_copy) { + if (c == '\\') { + char c2; + IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); + + /* get next character */ + c2 = copy_raw_buf[raw_buf_ptr]; + + /* + * If the following character is a newline or CRLF, + * skip the '\\'. + */ + if (c2 == '\n' || c2 == '\r' || + (c2 == '\r' && (raw_buf_ptr + 1) < copy_buf_len && copy_raw_buf[raw_buf_ptr + 1] == '\n')) { + continue; + } + } + } if (csv_mode) { /* From 4b17fdc1b6eb0291c99653ba669f88f6e5604552 Mon Sep 17 00:00:00 2001 From: ape Date: Fri, 8 Dec 2023 11:04:28 +0800 Subject: [PATCH 301/304] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E9=BE=99=E8=8A=AFloo?= =?UTF-8?q?ngarch64=E9=80=82=E9=85=8D=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build/script/cmake_package_mini.sh | 14 + build/script/loongarch64_lite_list | 916 +++++++++++ build/script/loongarch64_opengauss_list | 1448 +++++++++++++++++ build/script/utils/common.sh | 4 + build/script/utils/make_compile.sh | 23 +- cmake/src/build_options.cmake | 2 + config/config.guess | 30 + config/config.sub | 2 + configure | 53 +- src/Makefile.global.in | 23 +- src/common/backend/utils/misc/guc.cpp | 18 + .../backend/utils/misc/guc/guc_memory.cpp | 2 + .../backend/utils/misc/guc/guc_network.cpp | 2 + .../backend/utils/misc/guc/guc_resource.cpp | 2 + .../backend/utils/misc/guc/guc_security.cpp | 2 + src/common/backend/utils/misc/guc/guc_sql.cpp | 2 + .../backend/utils/misc/guc/guc_storage.cpp | 2 + src/common/port/CMakeLists.txt | 9 + src/common/port/Makefile | 17 +- src/gausskernel/CMakeLists.txt | 12 +- src/gausskernel/Makefile | 4 +- src/gausskernel/Makefile_for_llt | 12 +- src/gausskernel/cbb/CMakeLists.txt | 6 +- src/gausskernel/cbb/Makefile | 5 +- .../process/postmaster/pagewriter.cpp | 14 +- .../process/postmaster/postmaster.cpp | 10 + src/gausskernel/runtime/Makefile | 4 +- .../runtime/executor/instrument.cpp | 4 +- .../vecexecutor/vectorsonic/vsonichash.cpp | 25 +- .../storage/access/transam/double_write.cpp | 5 + .../access/transam/extreme_rto/dispatcher.cpp | 4 +- .../access/transam/extreme_rto/page_redo.cpp | 4 +- .../storage/access/transam/xlog.cpp | 125 +- src/gausskernel/storage/buffer/buf_init.cpp | 4 + .../storage/replication/dataqueue.cpp | 5 +- .../storage/replication/datareceiver.cpp | 3 + .../storage/replication/datasender.cpp | 7 +- .../storage/replication/walreceiver.cpp | 6 + .../storage/replication/walsender.cpp | 8 + src/include/access/extreme_rto/dispatcher.h | 2 +- src/include/access/extreme_rto/page_redo.h | 2 +- .../access/ondemand_extreme_rto/page_redo.h | 2 +- .../access/parallel_recovery/page_redo.h | 2 +- src/include/access/xlog.h | 2 +- .../communication/commproxy_interface.h | 4 +- src/include/pg_config.h.in | 8 + src/include/storage/lock/s_lock.h | 41 + src/include/utils/atomic.h | 2 + 48 files changed, 2798 insertions(+), 105 deletions(-) create mode 100644 build/script/loongarch64_lite_list create mode 100644 build/script/loongarch64_opengauss_list diff --git a/build/script/cmake_package_mini.sh b/build/script/cmake_package_mini.sh index 7e715c257..8c8e2e449 100644 --- a/build/script/cmake_package_mini.sh +++ b/build/script/cmake_package_mini.sh @@ -48,6 +48,8 @@ elif [ X"$kernel" == X"centos" ]; then dist_version="CentOS" elif [ X"$kernel" == X"openeuler" ]; then dist_version="openEuler" +elif [ X"$kernel" == X"kylin" ]; then + dist_version="kylin" else dist_version="Platform" fi @@ -271,6 +273,13 @@ read_mpp_version if [ "$gcc_version" == "7.3.0" ]; then gcc_version=${gcc_version:0:3} +elif [ "$gcc_version" == "8.3.0" ]; then + if [ $PLATFORM_ARCH == "loongarch64" ];then + gcc_version=${gcc_version:0:3} + else + echo "Only loongarch model use gcc8.3" + exit 1 + fi elif [ "$gcc_version" == "10.3.0" ] || [ "$gcc_version" == "10.3.1" ]; then gcc_version=${gcc_version:0:4} else @@ -462,6 +471,11 @@ function install_gaussdb() if [[ -e "/etc/openEuler-release" && "$(cat /etc/openEuler-release | awk '{print $3}')" == "22.03" ]]; then CMAKE_OPT="$CMAKE_OPT -DENABLE_OPENEULER_MAJOR=ON" fi + + if [ "${PLATFORM_ARCH}"x == "loongarch64"x ]; then + CMAKE_OPT="$CMAKE_OPT -DENABLE_BBOX=OFF -DENABLE_JEMALLOC=OFF" + fi + echo "CMAKE_OPT----> $CMAKE_OPT" echo "Begin run cmake for gaussdb server" >> "$LOG_FILE" 2>&1 echo "CMake options: ${CMAKE_OPT}" >> "$LOG_FILE" 2>&1 diff --git a/build/script/loongarch64_lite_list b/build/script/loongarch64_lite_list new file mode 100644 index 000000000..3d6e75783 --- /dev/null +++ b/build/script/loongarch64_lite_list @@ -0,0 +1,916 @@ +[server] +./bin/gsql +./bin/gaussdb +./bin/gstrace +./bin/gs_dump +./bin/gs_dumpall +./bin/gs_ctl +./bin/gs_initdb +./bin/gs_guc +./bin/gs_restore +./bin/pg_config +./bin/pg_controldata +./bin/gs_probackup +./bin/pg_resetxlog +./bin/alarmItem.conf +./bin/retry_errcodes.conf +./bin/cluster_guc.conf +./bin/lz4 +./bin/gs_plan_simulator.sh +./share/postgresql/db4ai +./share/postgresql/snowball_create.sql +./share/postgresql/pg_hba.conf.sample +./share/postgresql/gs_gazelle.conf.sample +./share/postgresql/pg_service.conf.sample +./share/postgresql/psqlrc.sample +./share/postgresql/conversion_create.sql +./share/postgresql/postgres.shdescription +./share/postgresql/pg_ident.conf.sample +./share/postgresql/postgres.description +./share/postgresql/postgresql.conf.sample +./share/postgresql/extension/plpgsql--1.0.sql +./share/postgresql/extension/hstore.control +./share/postgresql/extension/security_plugin.control +./share/postgresql/extension/security_plugin--1.0.sql +./share/postgresql/extension/dolphin.control +./share/postgresql/extension/dolphin--2.0.sql +./share/postgresql/extension/dolphin--1.0.sql +./share/postgresql/extension/dolphin--1.0--2.0.sql +./share/postgresql/extension/dolphin--2.0--1.0.sql +./share/postgresql/extension/dolphin--1.1--2.0.sql +./share/postgresql/extension/dolphin--2.0--1.1.sql +./share/postgresql/extension/openGauss_expr_dolphin.ir +./share/postgresql/extension/file_fdw--1.0.sql +./share/postgresql/extension/plpgsql.control +./share/postgresql/extension/dist_fdw.control +./share/postgresql/extension/dist_fdw--1.0.sql +./share/postgresql/extension/hstore--1.1.sql +./share/postgresql/extension/plpgsql--unpackaged--1.0.sql +./share/postgresql/extension/file_fdw.control +./share/postgresql/extension/hstore--unpackaged--1.0.sql +./share/postgresql/extension/hstore--1.0--1.1.sql +./share/postgresql/extension/log_fdw--1.0.sql +./share/postgresql/extension/log_fdw.control +./share/postgresql/timezone/GB-Eire +./share/postgresql/timezone/Turkey +./share/postgresql/timezone/Kwajalein +./share/postgresql/timezone/UCT +./share/postgresql/timezone/Mexico/BajaSur +./share/postgresql/timezone/Mexico/BajaNorte +./share/postgresql/timezone/Mexico/General +./share/postgresql/timezone/Japan +./share/postgresql/timezone/Israel +./share/postgresql/timezone/US/Eastern +./share/postgresql/timezone/US/Samoa +./share/postgresql/timezone/US/Michigan +./share/postgresql/timezone/US/Aleutian +./share/postgresql/timezone/US/Pacific +./share/postgresql/timezone/US/Pacific-New +./share/postgresql/timezone/US/Indiana-Starke +./share/postgresql/timezone/US/Mountain +./share/postgresql/timezone/US/East-Indiana +./share/postgresql/timezone/US/Hawaii +./share/postgresql/timezone/US/Arizona +./share/postgresql/timezone/US/Alaska +./share/postgresql/timezone/US/Central +./share/postgresql/timezone/Greenwich +./share/postgresql/timezone/Poland +./share/postgresql/timezone/CET +./share/postgresql/timezone/GMT-0 +./share/postgresql/timezone/Indian/Mauritius +./share/postgresql/timezone/Indian/Cocos +./share/postgresql/timezone/Indian/Reunion +./share/postgresql/timezone/Indian/Maldives +./share/postgresql/timezone/Indian/Comoro +./share/postgresql/timezone/Indian/Antananarivo +./share/postgresql/timezone/Indian/Christmas +./share/postgresql/timezone/Indian/Kerguelen +./share/postgresql/timezone/Indian/Chagos +./share/postgresql/timezone/Indian/Mayotte +./share/postgresql/timezone/Indian/Mahe +./share/postgresql/timezone/GMT0 +./share/postgresql/timezone/Antarctica/Palmer +./share/postgresql/timezone/Antarctica/Syowa +./share/postgresql/timezone/Antarctica/South_Pole +./share/postgresql/timezone/Antarctica/McMurdo +./share/postgresql/timezone/Antarctica/Rothera +./share/postgresql/timezone/Antarctica/Mawson +./share/postgresql/timezone/Antarctica/Casey +./share/postgresql/timezone/Antarctica/Davis +./share/postgresql/timezone/Antarctica/DumontDUrville +./share/postgresql/timezone/Antarctica/Vostok +./share/postgresql/timezone/Antarctica/Macquarie +./share/postgresql/timezone/ROK +./share/postgresql/timezone/Chile/EasterIsland +./share/postgresql/timezone/Chile/Continental +./share/postgresql/timezone/posixrules +./share/postgresql/timezone/Atlantic/Azores +./share/postgresql/timezone/Atlantic/St_Helena +./share/postgresql/timezone/Atlantic/Madeira +./share/postgresql/timezone/Atlantic/Jan_Mayen +./share/postgresql/timezone/Atlantic/Faroe +./share/postgresql/timezone/Atlantic/Stanley +./share/postgresql/timezone/Atlantic/Cape_Verde +./share/postgresql/timezone/Atlantic/Reykjavik +./share/postgresql/timezone/Atlantic/Canary +./share/postgresql/timezone/Atlantic/South_Georgia +./share/postgresql/timezone/Atlantic/Bermuda +./share/postgresql/timezone/Atlantic/Faeroe +./share/postgresql/timezone/Hongkong +./share/postgresql/timezone/Libya +./share/postgresql/timezone/Iceland +./share/postgresql/timezone/UTC +./share/postgresql/timezone/Australia/Darwin +./share/postgresql/timezone/Australia/North +./share/postgresql/timezone/Australia/NSW +./share/postgresql/timezone/Australia/Sydney +./share/postgresql/timezone/Australia/Hobart +./share/postgresql/timezone/Australia/LHI +./share/postgresql/timezone/Australia/Victoria +./share/postgresql/timezone/Australia/South +./share/postgresql/timezone/Australia/Melbourne +./share/postgresql/timezone/Australia/Lord_Howe +./share/postgresql/timezone/Australia/West +./share/postgresql/timezone/Australia/Brisbane +./share/postgresql/timezone/Australia/Perth +./share/postgresql/timezone/Australia/Eucla +./share/postgresql/timezone/Australia/Canberra +./share/postgresql/timezone/Australia/Queensland +./share/postgresql/timezone/Australia/Broken_Hill +./share/postgresql/timezone/Australia/Lindeman +./share/postgresql/timezone/Australia/ACT +./share/postgresql/timezone/Australia/Currie +./share/postgresql/timezone/Australia/Adelaide +./share/postgresql/timezone/Australia/Yancowinna +./share/postgresql/timezone/Australia/Tasmania +./share/postgresql/timezone/Jamaica +./share/postgresql/timezone/EST5EDT +./share/postgresql/timezone/MET +./share/postgresql/timezone/W-SU +./share/postgresql/timezone/Mideast/Riyadh87 +./share/postgresql/timezone/Mideast/Riyadh88 +./share/postgresql/timezone/Mideast/Riyadh89 +./share/postgresql/timezone/WET +./share/postgresql/timezone/ROC +./share/postgresql/timezone/Factory +./share/postgresql/timezone/EET +./share/postgresql/timezone/PST8PDT +./share/postgresql/timezone/Portugal +./share/postgresql/timezone/NZ +./share/postgresql/timezone/Brazil/West +./share/postgresql/timezone/Brazil/DeNoronha +./share/postgresql/timezone/Brazil/Acre +./share/postgresql/timezone/Brazil/East +./share/postgresql/timezone/EST +./share/postgresql/timezone/Egypt +./share/postgresql/timezone/Universal +./share/postgresql/timezone/Pacific/Enderbury +./share/postgresql/timezone/Pacific/Noumea +./share/postgresql/timezone/Pacific/Kwajalein +./share/postgresql/timezone/Pacific/Norfolk +./share/postgresql/timezone/Pacific/Nauru +./share/postgresql/timezone/Pacific/Efate +./share/postgresql/timezone/Pacific/Kosrae +./share/postgresql/timezone/Pacific/Galapagos +./share/postgresql/timezone/Pacific/Truk +./share/postgresql/timezone/Pacific/Fiji +./share/postgresql/timezone/Pacific/Auckland +./share/postgresql/timezone/Pacific/Samoa +./share/postgresql/timezone/Pacific/Port_Moresby +./share/postgresql/timezone/Pacific/Johnston +./share/postgresql/timezone/Pacific/Apia +./share/postgresql/timezone/Pacific/Tarawa +./share/postgresql/timezone/Pacific/Pitcairn +./share/postgresql/timezone/Pacific/Marquesas +./share/postgresql/timezone/Pacific/Chatham +./share/postgresql/timezone/Pacific/Tahiti +./share/postgresql/timezone/Pacific/Tongatapu +./share/postgresql/timezone/Pacific/Saipan +./share/postgresql/timezone/Pacific/Fakaofo +./share/postgresql/timezone/Pacific/Guam +./share/postgresql/timezone/Pacific/Niue +./share/postgresql/timezone/Pacific/Chuuk +./share/postgresql/timezone/Pacific/Easter +./share/postgresql/timezone/Pacific/Wallis +./share/postgresql/timezone/Pacific/Gambier +./share/postgresql/timezone/Pacific/Majuro +./share/postgresql/timezone/Pacific/Kiritimati +./share/postgresql/timezone/Pacific/Guadalcanal +./share/postgresql/timezone/Pacific/Funafuti +./share/postgresql/timezone/Pacific/Rarotonga +./share/postgresql/timezone/Pacific/Pago_Pago +./share/postgresql/timezone/Pacific/Midway +./share/postgresql/timezone/Pacific/Palau +./share/postgresql/timezone/Pacific/Honolulu +./share/postgresql/timezone/Pacific/Yap +./share/postgresql/timezone/Pacific/Pohnpei +./share/postgresql/timezone/Pacific/Wake +./share/postgresql/timezone/Pacific/Ponape +./share/postgresql/timezone/Iran +./share/postgresql/timezone/Etc/GMT-4 +./share/postgresql/timezone/Etc/GMT-9 +./share/postgresql/timezone/Etc/UCT +./share/postgresql/timezone/Etc/GMT-7 +./share/postgresql/timezone/Etc/Greenwich +./share/postgresql/timezone/Etc/GMT-0 +./share/postgresql/timezone/Etc/GMT-5 +./share/postgresql/timezone/Etc/GMT0 +./share/postgresql/timezone/Etc/GMT+7 +./share/postgresql/timezone/Etc/GMT-2 +./share/postgresql/timezone/Etc/GMT-10 +./share/postgresql/timezone/Etc/GMT+6 +./share/postgresql/timezone/Etc/GMT+1 +./share/postgresql/timezone/Etc/UTC +./share/postgresql/timezone/Etc/GMT-3 +./share/postgresql/timezone/Etc/GMT-1 +./share/postgresql/timezone/Etc/GMT-8 +./share/postgresql/timezone/Etc/GMT-11 +./share/postgresql/timezone/Etc/GMT+12 +./share/postgresql/timezone/Etc/GMT+10 +./share/postgresql/timezone/Etc/GMT-12 +./share/postgresql/timezone/Etc/GMT+5 +./share/postgresql/timezone/Etc/Universal +./share/postgresql/timezone/Etc/GMT-13 +./share/postgresql/timezone/Etc/GMT-6 +./share/postgresql/timezone/Etc/GMT+9 +./share/postgresql/timezone/Etc/GMT+3 +./share/postgresql/timezone/Etc/GMT-14 +./share/postgresql/timezone/Etc/GMT+4 +./share/postgresql/timezone/Etc/Zulu +./share/postgresql/timezone/Etc/GMT+2 +./share/postgresql/timezone/Etc/GMT+0 +./share/postgresql/timezone/Etc/GMT+8 +./share/postgresql/timezone/Etc/GMT+11 +./share/postgresql/timezone/Etc/GMT +./share/postgresql/timezone/Zulu +./share/postgresql/timezone/GMT+0 +./share/postgresql/timezone/Singapore +./share/postgresql/timezone/NZ-CHAT +./share/postgresql/timezone/Cuba +./share/postgresql/timezone/GB +./share/postgresql/timezone/Arctic/Longyearbyen +./share/postgresql/timezone/MST7MDT +./share/postgresql/timezone/PRC +./share/postgresql/timezone/Canada/Eastern +./share/postgresql/timezone/Canada/Yukon +./share/postgresql/timezone/Canada/Atlantic +./share/postgresql/timezone/Canada/Newfoundland +./share/postgresql/timezone/Canada/Saskatchewan +./share/postgresql/timezone/Canada/Pacific +./share/postgresql/timezone/Canada/Mountain +./share/postgresql/timezone/Canada/Central +./share/postgresql/timezone/CST6CDT +./share/postgresql/timezone/HST +./share/postgresql/timezone/America/Boa_Vista +./share/postgresql/timezone/America/New_York +./share/postgresql/timezone/America/Santarem +./share/postgresql/timezone/America/Boise +./share/postgresql/timezone/America/St_Lucia +./share/postgresql/timezone/America/Mendoza +./share/postgresql/timezone/America/Mexico_City +./share/postgresql/timezone/America/Chihuahua +./share/postgresql/timezone/America/Indianapolis +./share/postgresql/timezone/America/Virgin +./share/postgresql/timezone/America/Atka +./share/postgresql/timezone/America/Winnipeg +./share/postgresql/timezone/America/Hermosillo +./share/postgresql/timezone/America/Indiana/Indianapolis +./share/postgresql/timezone/America/Indiana/Tell_City +./share/postgresql/timezone/America/Indiana/Winamac +./share/postgresql/timezone/America/Indiana/Knox +./share/postgresql/timezone/America/Indiana/Vincennes +./share/postgresql/timezone/America/Indiana/Vevay +./share/postgresql/timezone/America/Indiana/Petersburg +./share/postgresql/timezone/America/Indiana/Marengo +./share/postgresql/timezone/America/Moncton +./share/postgresql/timezone/America/Campo_Grande +./share/postgresql/timezone/America/Guyana +./share/postgresql/timezone/America/Caracas +./share/postgresql/timezone/America/Maceio +./share/postgresql/timezone/America/Godthab +./share/postgresql/timezone/America/Thunder_Bay +./share/postgresql/timezone/America/Havana +./share/postgresql/timezone/America/Santiago +./share/postgresql/timezone/America/Los_Angeles +./share/postgresql/timezone/America/Buenos_Aires +./share/postgresql/timezone/America/Manaus +./share/postgresql/timezone/America/Bahia +./share/postgresql/timezone/America/North_Dakota/New_Salem +./share/postgresql/timezone/America/North_Dakota/Beulah +./share/postgresql/timezone/America/North_Dakota/Center +./share/postgresql/timezone/America/Bahia_Banderas +./share/postgresql/timezone/America/Edmonton +./share/postgresql/timezone/America/Tegucigalpa +./share/postgresql/timezone/America/Rankin_Inlet +./share/postgresql/timezone/America/Monterrey +./share/postgresql/timezone/America/Cambridge_Bay +./share/postgresql/timezone/America/Porto_Velho +./share/postgresql/timezone/America/Antigua +./share/postgresql/timezone/America/Atikokan +./share/postgresql/timezone/America/Vancouver +./share/postgresql/timezone/America/Anchorage +./share/postgresql/timezone/America/Port-au-Prince +./share/postgresql/timezone/America/Lima +./share/postgresql/timezone/America/Grenada +./share/postgresql/timezone/America/Creston +./share/postgresql/timezone/America/La_Paz +./share/postgresql/timezone/America/Panama +./share/postgresql/timezone/America/Blanc-Sablon +./share/postgresql/timezone/America/Cayenne +./share/postgresql/timezone/America/Santo_Domingo +./share/postgresql/timezone/America/Grand_Turk +./share/postgresql/timezone/America/Toronto +./share/postgresql/timezone/America/Rainy_River +./share/postgresql/timezone/America/Merida +./share/postgresql/timezone/America/Port_of_Spain +./share/postgresql/timezone/America/Nipigon +./share/postgresql/timezone/America/Jamaica +./share/postgresql/timezone/America/Rosario +./share/postgresql/timezone/America/Dawson_Creek +./share/postgresql/timezone/America/Belize +./share/postgresql/timezone/America/Costa_Rica +./share/postgresql/timezone/America/Barbados +./share/postgresql/timezone/America/Danmarkshavn +./share/postgresql/timezone/America/Argentina/La_Rioja +./share/postgresql/timezone/America/Argentina/Mendoza +./share/postgresql/timezone/America/Argentina/Buenos_Aires +./share/postgresql/timezone/America/Argentina/Tucuman +./share/postgresql/timezone/America/Argentina/Ushuaia +./share/postgresql/timezone/America/Argentina/Catamarca +./share/postgresql/timezone/America/Argentina/ComodRivadavia +./share/postgresql/timezone/America/Argentina/Jujuy +./share/postgresql/timezone/America/Argentina/Cordoba +./share/postgresql/timezone/America/Argentina/San_Luis +./share/postgresql/timezone/America/Argentina/Rio_Gallegos +./share/postgresql/timezone/America/Argentina/Salta +./share/postgresql/timezone/America/Argentina/San_Juan +./share/postgresql/timezone/America/Pangnirtung +./share/postgresql/timezone/America/Anguilla +./share/postgresql/timezone/America/Curacao +./share/postgresql/timezone/America/Cancun +./share/postgresql/timezone/America/Montreal +./share/postgresql/timezone/America/Shiprock +./share/postgresql/timezone/America/Thule +./share/postgresql/timezone/America/Scoresbysund +./share/postgresql/timezone/America/Catamarca +./share/postgresql/timezone/America/Sao_Paulo +./share/postgresql/timezone/America/Sitka +./share/postgresql/timezone/America/Asuncion +./share/postgresql/timezone/America/Regina +./share/postgresql/timezone/America/St_Johns +./share/postgresql/timezone/America/Montevideo +./share/postgresql/timezone/America/Eirunepe +./share/postgresql/timezone/America/Denver +./share/postgresql/timezone/America/Metlakatla +./share/postgresql/timezone/America/Araguaina +./share/postgresql/timezone/America/Juneau +./share/postgresql/timezone/America/Marigot +./share/postgresql/timezone/America/Menominee +./share/postgresql/timezone/America/Glace_Bay +./share/postgresql/timezone/America/Tijuana +./share/postgresql/timezone/America/Detroit +./share/postgresql/timezone/America/Belem +./share/postgresql/timezone/America/Jujuy +./share/postgresql/timezone/America/St_Thomas +./share/postgresql/timezone/America/Resolute +./share/postgresql/timezone/America/Cuiaba +./share/postgresql/timezone/America/Halifax +./share/postgresql/timezone/America/St_Barthelemy +./share/postgresql/timezone/America/Guatemala +./share/postgresql/timezone/America/Nassau +./share/postgresql/timezone/America/St_Kitts +./share/postgresql/timezone/America/Cordoba +./share/postgresql/timezone/America/Miquelon +./share/postgresql/timezone/America/Bogota +./share/postgresql/timezone/America/Rio_Branco +./share/postgresql/timezone/America/Ensenada +./share/postgresql/timezone/America/Yakutat +./share/postgresql/timezone/America/Noronha +./share/postgresql/timezone/America/Kentucky/Monticello +./share/postgresql/timezone/America/Kentucky/Louisville +./share/postgresql/timezone/America/Porto_Acre +./share/postgresql/timezone/America/Santa_Isabel +./share/postgresql/timezone/America/El_Salvador +./share/postgresql/timezone/America/Yellowknife +./share/postgresql/timezone/America/Cayman +./share/postgresql/timezone/America/Whitehorse +./share/postgresql/timezone/America/Ojinaga +./share/postgresql/timezone/America/Aruba +./share/postgresql/timezone/America/Nome +./share/postgresql/timezone/America/Fortaleza +./share/postgresql/timezone/America/Martinique +./share/postgresql/timezone/America/Recife +./share/postgresql/timezone/America/Knox_IN +./share/postgresql/timezone/America/Guayaquil +./share/postgresql/timezone/America/Goose_Bay +./share/postgresql/timezone/America/Iqaluit +./share/postgresql/timezone/America/Matamoros +./share/postgresql/timezone/America/Lower_Princes +./share/postgresql/timezone/America/Louisville +./share/postgresql/timezone/America/Coral_Harbour +./share/postgresql/timezone/America/Phoenix +./share/postgresql/timezone/America/Guadeloupe +./share/postgresql/timezone/America/Mazatlan +./share/postgresql/timezone/America/Swift_Current +./share/postgresql/timezone/America/Paramaribo +./share/postgresql/timezone/America/Dominica +./share/postgresql/timezone/America/Kralendijk +./share/postgresql/timezone/America/Montserrat +./share/postgresql/timezone/America/St_Vincent +./share/postgresql/timezone/America/Fort_Wayne +./share/postgresql/timezone/America/Dawson +./share/postgresql/timezone/America/Inuvik +./share/postgresql/timezone/America/Adak +./share/postgresql/timezone/America/Managua +./share/postgresql/timezone/America/Puerto_Rico +./share/postgresql/timezone/America/Tortola +./share/postgresql/timezone/America/Chicago +./share/postgresql/timezone/Africa/Lome +./share/postgresql/timezone/Africa/Brazzaville +./share/postgresql/timezone/Africa/Khartoum +./share/postgresql/timezone/Africa/Ceuta +./share/postgresql/timezone/Africa/Djibouti +./share/postgresql/timezone/Africa/Lagos +./share/postgresql/timezone/Africa/Accra +./share/postgresql/timezone/Africa/El_Aaiun +./share/postgresql/timezone/Africa/Malabo +./share/postgresql/timezone/Africa/Windhoek +./share/postgresql/timezone/Africa/Tripoli +./share/postgresql/timezone/Africa/Bissau +./share/postgresql/timezone/Africa/Blantyre +./share/postgresql/timezone/Africa/Kinshasa +./share/postgresql/timezone/Africa/Porto-Novo +./share/postgresql/timezone/Africa/Nairobi +./share/postgresql/timezone/Africa/Ouagadougou +./share/postgresql/timezone/Africa/Asmera +./share/postgresql/timezone/Africa/Cairo +./share/postgresql/timezone/Africa/Lubumbashi +./share/postgresql/timezone/Africa/Tunis +./share/postgresql/timezone/Africa/Dar_es_Salaam +./share/postgresql/timezone/Africa/Casablanca +./share/postgresql/timezone/Africa/Algiers +./share/postgresql/timezone/Africa/Mbabane +./share/postgresql/timezone/Africa/Monrovia +./share/postgresql/timezone/Africa/Nouakchott +./share/postgresql/timezone/Africa/Banjul +./share/postgresql/timezone/Africa/Kampala +./share/postgresql/timezone/Africa/Conakry +./share/postgresql/timezone/Africa/Mogadishu +./share/postgresql/timezone/Africa/Ndjamena +./share/postgresql/timezone/Africa/Niamey +./share/postgresql/timezone/Africa/Lusaka +./share/postgresql/timezone/Africa/Addis_Ababa +./share/postgresql/timezone/Africa/Sao_Tome +./share/postgresql/timezone/Africa/Abidjan +./share/postgresql/timezone/Africa/Harare +./share/postgresql/timezone/Africa/Asmara +./share/postgresql/timezone/Africa/Douala +./share/postgresql/timezone/Africa/Freetown +./share/postgresql/timezone/Africa/Libreville +./share/postgresql/timezone/Africa/Luanda +./share/postgresql/timezone/Africa/Maseru +./share/postgresql/timezone/Africa/Gaborone +./share/postgresql/timezone/Africa/Maputo +./share/postgresql/timezone/Africa/Timbuktu +./share/postgresql/timezone/Africa/Bangui +./share/postgresql/timezone/Africa/Bamako +./share/postgresql/timezone/Africa/Dakar +./share/postgresql/timezone/Africa/Juba +./share/postgresql/timezone/Africa/Bujumbura +./share/postgresql/timezone/Africa/Johannesburg +./share/postgresql/timezone/Africa/Kigali +./share/postgresql/timezone/Eire +./share/postgresql/timezone/Europe/Vaduz +./share/postgresql/timezone/Europe/Podgorica +./share/postgresql/timezone/Europe/Rome +./share/postgresql/timezone/Europe/Vienna +./share/postgresql/timezone/Europe/Dublin +./share/postgresql/timezone/Europe/Zurich +./share/postgresql/timezone/Europe/London +./share/postgresql/timezone/Europe/Monaco +./share/postgresql/timezone/Europe/Sofia +./share/postgresql/timezone/Europe/Uzhgorod +./share/postgresql/timezone/Europe/Minsk +./share/postgresql/timezone/Europe/Malta +./share/postgresql/timezone/Europe/Busingen +./share/postgresql/timezone/Europe/Gibraltar +./share/postgresql/timezone/Europe/Volgograd +./share/postgresql/timezone/Europe/Budapest +./share/postgresql/timezone/Europe/Vatican +./share/postgresql/timezone/Europe/Luxembourg +./share/postgresql/timezone/Europe/Chisinau +./share/postgresql/timezone/Europe/Nicosia +./share/postgresql/timezone/Europe/Warsaw +./share/postgresql/timezone/Europe/San_Marino +./share/postgresql/timezone/Europe/Copenhagen +./share/postgresql/timezone/Europe/Ljubljana +./share/postgresql/timezone/Europe/Athens +./share/postgresql/timezone/Europe/Skopje +./share/postgresql/timezone/Europe/Andorra +./share/postgresql/timezone/Europe/Kaliningrad +./share/postgresql/timezone/Europe/Amsterdam +./share/postgresql/timezone/Europe/Guernsey +./share/postgresql/timezone/Europe/Isle_of_Man +./share/postgresql/timezone/Europe/Tirane +./share/postgresql/timezone/Europe/Jersey +./share/postgresql/timezone/Europe/Madrid +./share/postgresql/timezone/Europe/Helsinki +./share/postgresql/timezone/Europe/Riga +./share/postgresql/timezone/Europe/Zagreb +./share/postgresql/timezone/Europe/Bratislava +./share/postgresql/timezone/Europe/Prague +./share/postgresql/timezone/Europe/Tallinn +./share/postgresql/timezone/Europe/Stockholm +./share/postgresql/timezone/Europe/Tiraspol +./share/postgresql/timezone/Europe/Belgrade +./share/postgresql/timezone/Europe/Bucharest +./share/postgresql/timezone/Europe/Vilnius +./share/postgresql/timezone/Europe/Sarajevo +./share/postgresql/timezone/Europe/Belfast +./share/postgresql/timezone/Europe/Zaporozhye +./share/postgresql/timezone/Europe/Oslo +./share/postgresql/timezone/Europe/Mariehamn +./share/postgresql/timezone/Europe/Moscow +./share/postgresql/timezone/Europe/Brussels +./share/postgresql/timezone/Europe/Paris +./share/postgresql/timezone/Europe/Istanbul +./share/postgresql/timezone/Europe/Simferopol +./share/postgresql/timezone/Europe/Lisbon +./share/postgresql/timezone/Europe/Berlin +./share/postgresql/timezone/Europe/Kiev +./share/postgresql/timezone/Europe/Samara +./share/postgresql/timezone/MST +./share/postgresql/timezone/Asia/Khandyga +./share/postgresql/timezone/Asia/Manila +./share/postgresql/timezone/Asia/Novokuznetsk +./share/postgresql/timezone/Asia/Baghdad +./share/postgresql/timezone/Asia/Macau +./share/postgresql/timezone/Asia/Urumqi +./share/postgresql/timezone/Asia/Ujung_Pandang +./share/postgresql/timezone/Asia/Ulan_Bator +./share/postgresql/timezone/Asia/Bishkek +./share/postgresql/timezone/Asia/Qatar +./share/postgresql/timezone/Asia/Qyzylorda +./share/postgresql/timezone/Asia/Calcutta +./share/postgresql/timezone/Asia/Riyadh87 +./share/postgresql/timezone/Asia/Dushanbe +./share/postgresql/timezone/Asia/Yekaterinburg +./share/postgresql/timezone/Asia/Dhaka +./share/postgresql/timezone/Asia/Jakarta +./share/postgresql/timezone/Asia/Shanghai +./share/postgresql/timezone/Asia/Ulaanbaatar +./share/postgresql/timezone/Asia/Jerusalem +./share/postgresql/timezone/Asia/Ashkhabad +./share/postgresql/timezone/Asia/Tokyo +./share/postgresql/timezone/Asia/Macao +./share/postgresql/timezone/Asia/Krasnoyarsk +./share/postgresql/timezone/Asia/Saigon +./share/postgresql/timezone/Asia/Omsk +./share/postgresql/timezone/Asia/Damascus +./share/postgresql/timezone/Asia/Phnom_Penh +./share/postgresql/timezone/Asia/Bangkok +./share/postgresql/timezone/Asia/Kamchatka +./share/postgresql/timezone/Asia/Choibalsan +./share/postgresql/timezone/Asia/Ust-Nera +./share/postgresql/timezone/Asia/Aden +./share/postgresql/timezone/Asia/Vientiane +./share/postgresql/timezone/Asia/Sakhalin +./share/postgresql/timezone/Asia/Ashgabat +./share/postgresql/timezone/Asia/Katmandu +./share/postgresql/timezone/Asia/Almaty +./share/postgresql/timezone/Asia/Baku +./share/postgresql/timezone/Asia/Nicosia +./share/postgresql/timezone/Asia/Riyadh88 +./share/postgresql/timezone/Asia/Kashgar +./share/postgresql/timezone/Asia/Riyadh89 +./share/postgresql/timezone/Asia/Taipei +./share/postgresql/timezone/Asia/Tehran +./share/postgresql/timezone/Asia/Kabul +./share/postgresql/timezone/Asia/Samarkand +./share/postgresql/timezone/Asia/Kuala_Lumpur +./share/postgresql/timezone/Asia/Tashkent +./share/postgresql/timezone/Asia/Thimbu +./share/postgresql/timezone/Asia/Thimphu +./share/postgresql/timezone/Asia/Yerevan +./share/postgresql/timezone/Asia/Chungking +./share/postgresql/timezone/Asia/Hebron +./share/postgresql/timezone/Asia/Karachi +./share/postgresql/timezone/Asia/Kolkata +./share/postgresql/timezone/Asia/Aqtobe +./share/postgresql/timezone/Asia/Muscat +./share/postgresql/timezone/Asia/Hong_Kong +./share/postgresql/timezone/Asia/Chongqing +./share/postgresql/timezone/Asia/Oral +./share/postgresql/timezone/Asia/Pontianak +./share/postgresql/timezone/Asia/Colombo +./share/postgresql/timezone/Asia/Pyongyang +./share/postgresql/timezone/Asia/Hovd +./share/postgresql/timezone/Asia/Kuwait +./share/postgresql/timezone/Asia/Anadyr +./share/postgresql/timezone/Asia/Kathmandu +./share/postgresql/timezone/Asia/Irkutsk +./share/postgresql/timezone/Asia/Bahrain +./share/postgresql/timezone/Asia/Dubai +./share/postgresql/timezone/Asia/Jayapura +./share/postgresql/timezone/Asia/Riyadh +./share/postgresql/timezone/Asia/Ho_Chi_Minh +./share/postgresql/timezone/Asia/Singapore +./share/postgresql/timezone/Asia/Tel_Aviv +./share/postgresql/timezone/Asia/Dili +./share/postgresql/timezone/Asia/Rangoon +./share/postgresql/timezone/Asia/Harbin +./share/postgresql/timezone/Asia/Yakutsk +./share/postgresql/timezone/Asia/Magadan +./share/postgresql/timezone/Asia/Amman +./share/postgresql/timezone/Asia/Kuching +./share/postgresql/timezone/Asia/Novosibirsk +./share/postgresql/timezone/Asia/Seoul +./share/postgresql/timezone/Asia/Dacca +./share/postgresql/timezone/Asia/Vladivostok +./share/postgresql/timezone/Asia/Istanbul +./share/postgresql/timezone/Asia/Beirut +./share/postgresql/timezone/Asia/Aqtau +./share/postgresql/timezone/Asia/Brunei +./share/postgresql/timezone/Asia/Gaza +./share/postgresql/timezone/Asia/Tbilisi +./share/postgresql/timezone/Asia/Makassar +./share/postgresql/timezone/Asia/Beijing +./share/postgresql/timezone/Navajo +./share/postgresql/timezone/GMT +./share/postgresql/system_views.sql +./share/postgresql/private_system_views.sql +./share/postgresql/performance_views.sql +./share/postgresql/sql_features.txt +./share/postgresql/pg_cast_oid.txt +./share/postgresql/recovery.conf.sample +./share/postgresql/tsearch_data/english.stop +./share/postgresql/tsearch_data/dutch.stop +./share/postgresql/tsearch_data/hungarian.stop +./share/postgresql/tsearch_data/french.stop +./share/postgresql/tsearch_data/synonym_sample.syn +./share/postgresql/tsearch_data/turkish.stop +./share/postgresql/tsearch_data/portuguese.stop +./share/postgresql/tsearch_data/spanish.stop +./share/postgresql/tsearch_data/hunspell_sample.affix +./share/postgresql/tsearch_data/ispell_sample.affix +./share/postgresql/tsearch_data/danish.stop +./share/postgresql/tsearch_data/german.stop +./share/postgresql/tsearch_data/thesaurus_sample.ths +./share/postgresql/tsearch_data/norwegian.stop +./share/postgresql/tsearch_data/finnish.stop +./share/postgresql/tsearch_data/russian.stop +./share/postgresql/tsearch_data/swedish.stop +./share/postgresql/tsearch_data/ispell_sample.dict +./share/postgresql/tsearch_data/italian.stop +./share/postgresql/information_schema.sql +./share/postgresql/timezonesets/Antarctica.txt +./share/postgresql/timezonesets/Australia.txt +./share/postgresql/timezonesets/Europe.txt +./share/postgresql/timezonesets/America.txt +./share/postgresql/timezonesets/Australia +./share/postgresql/timezonesets/Indian.txt +./share/postgresql/timezonesets/India +./share/postgresql/timezonesets/Pacific.txt +./share/postgresql/timezonesets/Atlantic.txt +./share/postgresql/timezonesets/Africa.txt +./share/postgresql/timezonesets/Asia.txt +./share/postgresql/timezonesets/Default +./share/postgresql/timezonesets/Etc.txt +./share/postgresql/postgres.bki +./share/sslcert/gsql/openssl.cnf +./lib/libnuma.so* +./lib/libpq.so +./lib/libpq.so.5 +./lib/libpq.so.5.5 +./lib/libssl.so* +./lib/libcrypto.so* +./lib/libcgroup.so* +./lib/libz.so* +./lib/liblz4.so* +./lib/libcjson.so* +./lib/libcjson_utils.so* +./lib/libstdc++.so.6 +./lib/libgcc_s.so.1 +./lib/libgomp.so* +./lib/libdcf.so +./lib/libzstd.so* +./lib/libcurl.so* +./lib/libxgboost.so +./lib/libpagecompression.so* +./lib/postgresql/latin2_and_win1250.so +./lib/postgresql/euc2004_sjis2004.so +./lib/postgresql/euc_kr_and_mic.so +./lib/postgresql/utf8_and_uhc.so +./lib/postgresql/euc_tw_and_big5.so +./lib/postgresql/cyrillic_and_mic.so +./lib/postgresql/utf8_and_johab.so +./lib/postgresql/utf8_and_gb18030.so +./lib/postgresql/pgxs/config/install-sh +./lib/postgresql/euc_cn_and_mic.so +./lib/postgresql/latin_and_mic.so +./lib/postgresql/utf8_and_sjis2004.so +./lib/postgresql/utf8_and_euc_jp.so +./lib/postgresql/utf8_and_sjis.so +./lib/postgresql/utf8_and_cyrillic.so +./lib/postgresql/hstore.so +./lib/postgresql/packages.so +./lib/postgresql/utf8_and_euc_kr.so +./lib/postgresql/ascii_and_mic.so +./lib/postgresql/utf8_and_iso8859_1.so +./lib/postgresql/euc_jp_and_sjis.so +./lib/postgresql/dict_snowball.so +./lib/postgresql/utf8_and_ascii.so +./lib/postgresql/utf8_and_euc_tw.so +./lib/postgresql/utf8_and_iso8859.so +./lib/postgresql/utf8_and_win.so +./lib/postgresql/utf8_and_euc_cn.so +./lib/postgresql/utf8_and_gbk.so +./lib/postgresql/utf8_and_euc2004.so +./lib/postgresql/utf8_and_big5.so +./lib/postgresql/mppdb_decoding.so +./lib/postgresql/sql_decoding.so +./lib/postgresql/pg_plugin +./lib/postgresql/proc_srclib +./lib/postgresql/security_plugin.so +./lib/postgresql/dolphin.so +./lib/postgresql/pg_upgrade_support.so +./lib/postgresql/latin2_and_win1250.so +./lib/postgresql/euc2004_sjis2004.so +./lib/postgresql/pgoutput.so +./include/postgresql/server/postgres_ext.h +./include/postgresql/server/pg_config_os.h +./include/postgresql/server/pgtime.h +./include/postgresql/server/datatypes.h +./include/postgresql/server/client_logic/client_logic_enums.h +./include/postgresql/server/libpq/libpq-fs.h +./include/postgresql/server/nodes/primnodes.h +./include/postgresql/server/nodes/parsenodes.h +./include/postgresql/server/nodes/parsenodes_common.h +./include/postgresql/server/nodes/bitmapset.h +./include/postgresql/server/nodes/pg_list.h +./include/postgresql/server/nodes/value.h +./include/postgresql/server/nodes/nodes.h +./include/postgresql/server/utils/sortsupport.h +./include/postgresql/server/utils/varbit.h +./include/postgresql/server/utils/spccache.h +./include/postgresql/server/utils/rangetypes.h +./include/postgresql/server/utils/plpgsql.h +./include/postgresql/server/utils/memtrack.h +./include/postgresql/server/utils/pg_locale.h +./include/postgresql/server/utils/tzparser.h +./include/postgresql/server/utils/syscall_lock.h +./include/postgresql/server/utils/partitionmap.h +./include/postgresql/server/utils/array.h +./include/postgresql/server/utils/relmapper.h +./include/postgresql/server/utils/hsearch.h +./include/postgresql/server/utils/xml.h +./include/postgresql/server/utils/bytea.h +./include/postgresql/server/utils/relcache.h +./include/postgresql/server/utils/pg_rusage.h +./include/postgresql/server/utils/numeric.h +./include/postgresql/server/utils/mmpool.h +./include/postgresql/server/utils/nabstime.h +./include/postgresql/server/utils/fmgrtab.h +./include/postgresql/server/utils/snapmgr.h +./include/postgresql/server/utils/syscache.h +./include/postgresql/server/utils/logtape.h +./include/postgresql/server/utils/datum.h +./include/postgresql/server/utils/guc_tables.h +./include/postgresql/server/utils/snapshot.h +./include/postgresql/server/utils/geo_decls.h +./include/postgresql/server/utils/errcodes.h +./include/postgresql/server/utils/inval.h +./include/postgresql/server/utils/help_config.h +./include/postgresql/server/utils/distribute_test.h +./include/postgresql/server/utils/aiomem.h +./include/postgresql/server/utils/tuplestore.h +./include/postgresql/server/utils/rbtree.h +./include/postgresql/server/utils/gs_bitmap.h +./include/postgresql/server/utils/tuplesort.h +./include/postgresql/server/utils/ps_status.h +./include/postgresql/server/utils/palloc.h +./include/postgresql/server/utils/reltrigger.h +./include/postgresql/server/utils/acl.h +./include/postgresql/server/utils/ascii.h +./include/postgresql/server/utils/selfuncs.h +./include/postgresql/server/utils/json.h +./include/postgresql/server/utils/portal.h +./include/postgresql/server/utils/atomic.h +./include/postgresql/server/utils/elog.h +./include/postgresql/server/utils/date.h +./include/postgresql/server/utils/plancache.h +./include/postgresql/server/utils/int8.h +./include/postgresql/server/utils/timestamp.h +./include/postgresql/server/utils/bloom_filter.h +./include/postgresql/server/utils/fmgroids.h +./include/postgresql/server/utils/pg_crc_tables.h +./include/postgresql/server/utils/probes.h +./include/postgresql/server/utils/datetime.h +./include/postgresql/server/utils/inet.h +./include/postgresql/server/utils/pg_lzcompress.h +./include/postgresql/server/utils/pg_crc.h +./include/postgresql/server/utils/attoptcache.h +./include/postgresql/server/utils/dynahash.h +./include/postgresql/server/utils/rel.h +./include/postgresql/server/utils/partcache.h +./include/postgresql/server/utils/lsyscache.h +./include/postgresql/server/utils/memutils.h +./include/postgresql/server/utils/memprot.h +./include/postgresql/server/utils/uuid.h +./include/postgresql/server/utils/combocid.h +./include/postgresql/server/utils/builtins.h +./include/postgresql/server/utils/guc.h +./include/postgresql/server/utils/dfs_vector.h +./include/postgresql/server/utils/dynamic_loader.h +./include/postgresql/server/utils/resowner.h +./include/postgresql/server/utils/aes.h +./include/postgresql/server/utils/cash.h +./include/postgresql/server/utils/typcache.h +./include/postgresql/server/utils/formatting.h +./include/postgresql/server/utils/partitionkey.h +./include/postgresql/server/utils/aset.h +./include/postgresql/server/utils/catcache.h +./include/postgresql/server/utils/atomic_arm.h +./include/postgresql/server/utils/oidrbtree.h +./include/postgresql/server/datatype/timestamp.h +./include/postgresql/server/access/rmgr.h +./include/postgresql/server/access/xlogreader.h +./include/postgresql/server/access/xlog_basic.h +./include/postgresql/server/access/tupdesc.h +./include/postgresql/server/access/rmgrlist.h +./include/postgresql/server/access/htup.h +./include/postgresql/server/access/xlogdefs.h +./include/postgresql/server/access/attnum.h +./include/postgresql/server/access/tupmacs.h +./include/postgresql/server/access/xlogrecord.h +./include/postgresql/server/tde_key_management/data_common.h +./include/postgresql/server/tcop/dest.h +./include/postgresql/server/catalog/pg_type.h +./include/postgresql/server/catalog/pg_attribute.h +./include/postgresql/server/catalog/genbki.h +./include/postgresql/server/gs_thread.h +./include/postgresql/server/port/pg_bswap.h +./include/postgresql/server/port/pg_crc32c.h +./include/postgresql/server/securec.h +./include/postgresql/server/securectype.h +./include/postgresql/server/storage/off.h +./include/postgresql/server/storage/buf/block.h +./include/postgresql/server/storage/item/item.h +./include/postgresql/server/storage/smgr/relfilenode.h +./include/postgresql/server/storage/buf/bufpage.h +./include/postgresql/server/storage/spin.h +./include/postgresql/server/storage/buf/buf.h +./include/postgresql/server/storage/item/itemid.h +./include/postgresql/server/storage/lock/pg_sema.h +./include/postgresql/server/storage/item/itemptr.h +./include/postgresql/server/storage/lock/s_lock.h +./include/postgresql/server/storage/backendid.h +./include/postgresql/server/storage/lock/lock.h +./include/postgresql/server/storage/lock/lwlock.h +./include/postgresql/server/storage/lwlocknames.h +./include/postgresql/server/storage/barrier.h +./include/postgresql/server/storage/shmem.h +./include/postgresql/server/pg_config.h +./include/postgresql/server/lib/stringinfo.h +./include/postgresql/server/fmgr.h +./include/postgresql/server/fmgr/fmgr_comp.h +./include/postgresql/server/fmgr/fmgr_core.h +./include/postgresql/server/gs_threadlocal.h +./include/postgresql/server/postgres.h +./include/postgresql/server/executor/tuptable.h +./include/postgresql/server/pg_config_manual.h +./include/postgresql/server/mb/pg_wchar.h +./include/postgresql/server/c.h +./include/postgresql/server/port.h +./include/postgresql/server/utils/be_module.h +./include/postgresql/server/nodes/params.h +./include/postgresql/server/securec_check.h +./include/postgresql/server/nodes/memnodes.h +./include/postgresql/server/access/skey.h +./include/postgresql/server/lib/dllist.h +./include/postgresql/server/lib/ilist.h +./include/postgresql/server/pgxc/locator.h +./include/postgresql/server/gstrace/gstrace_infra.h +./include/postgresql/server/db4ai/db4ai.h +[libpq] +./lib/libpq.so +./lib/libpq.so.5 +./lib/libpq.so.5.5 +./lib/libcrypto.so* +./lib/libssl.so* +[header] +./include/libpq/libpq-fs.h +./include/libpq-fe.h +./include/postgres_ext.h +./include/gs_thread.h +./include/gs_threadlocal.h +./include/pg_config.h +./include/pg_config_manual.h +./include/pg_config_os.h +./include/c.h +./include/port.h +./include/libpq-int.h +./include/pqcomm.h +./include/pqexpbuffer.h +[version] +5.1.0 diff --git a/build/script/loongarch64_opengauss_list b/build/script/loongarch64_opengauss_list new file mode 100644 index 000000000..e776d1ee9 --- /dev/null +++ b/build/script/loongarch64_opengauss_list @@ -0,0 +1,1448 @@ +[server] +./bin/dsscmd +./bin/dssserver +./bin/perctrl +./bin/dms_contrl.sh +./bin/dss_clear.sh +./bin/dss_contrl.sh +./bin/gsql +./bin/gaussdb +./bin/gstrace +./bin/gs_basebackup +./bin/gs_probackup +./bin/gs_tar +./bin/gs_encrypt +./bin/gs_dump +./bin/gs_dumpall +./bin/gs_ctl +./bin/gs_initdb +./bin/gs_guc +./bin/encrypt +./bin/openssl +./bin/gs_restore +./bin/gs_cgroup +./bin/openssl +./bin/pg_config +./bin/pg_controldata +./bin/pg_format_cu +./bin/pg_resetxlog +./bin/pg_recvlogical +./bin/alarmItem.conf +./bin/retry_errcodes.conf +./bin/cluster_guc.conf +./bin/bind_net_irq.sh +./bin/setArmOptimization.sh +./bin/krb5kdc +./bin/klist +./bin/kinit +./bin/kdestroy +./bin/kdb5_util +./bin/kadmin.local +./bin/lz4 +./bin/kadmind +./bin/gs_dbmind +./bin/constant +./bin/server.key.cipher +./bin/server.key.rand +./bin/gs_plan_simulator.sh +./bin/pg_xlogdump +./bin/pagehack +./bin/assessment_database +./etc/kerberos/kadm5.acl +./etc/kerberos/kdc.conf +./etc/kerberos/krb5.conf +./etc/kerberos/mppdb-site.xml +./share/postgresql/tmp/udstools.py +./share/postgresql/db4ai +./share/postgresql/snowball_create.sql +./share/postgresql/pg_hba.conf.sample +./share/postgresql/gs_gazelle.conf.sample +./share/postgresql/pg_service.conf.sample +./share/postgresql/psqlrc.sample +./share/postgresql/conversion_create.sql +./share/postgresql/postgres.shdescription +./share/postgresql/pg_ident.conf.sample +./share/postgresql/postgres.description +./share/postgresql/postgresql.conf.sample +./share/postgresql/mot.conf.sample +./share/postgresql/extension/plpgsql--1.0.sql +./share/postgresql/extension/hstore.control +./share/postgresql/extension/security_plugin.control +./share/postgresql/extension/security_plugin--1.0.sql +./share/postgresql/extension/ndpplugin.control +./share/postgresql/extension/ndpplugin--1.0.sql +./share/postgresql/extension/dolphin.control +./share/postgresql/extension/dolphin--2.0.sql +./share/postgresql/extension/dolphin--1.0.sql +./share/postgresql/extension/dolphin--1.0--2.0.sql +./share/postgresql/extension/dolphin--2.0--1.0.sql +./share/postgresql/extension/dolphin--1.1--2.0.sql +./share/postgresql/extension/dolphin--2.0--1.1.sql +./share/postgresql/extension/openGauss_expr_dolphin.ir +./share/postgresql/extension/assessment--1.0.sql +./share/postgresql/extension/assessment.control +./share/postgresql/extension/file_fdw--1.0.sql +./share/postgresql/extension/plpgsql.control +./share/postgresql/extension/dist_fdw.control +./share/postgresql/extension/dist_fdw--1.0.sql +./share/postgresql/extension/hstore--1.1.sql +./share/postgresql/extension/plpgsql--unpackaged--1.0.sql +./share/postgresql/extension/file_fdw.control +./share/postgresql/extension/hstore--unpackaged--1.0.sql +./share/postgresql/extension/hstore--1.0--1.1.sql +./share/postgresql/extension/log_fdw--1.0.sql +./share/postgresql/extension/log_fdw.control +./share/postgresql/extension/mot_fdw--1.0.sql +./share/postgresql/extension/mot_fdw.control +./share/postgresql/extension/postgres_fdw--1.0.sql +./share/postgresql/extension/postgres_fdw.control +./share/postgresql/extension/dblink--1.0.sql +./share/postgresql/extension/dblink--unpackaged--1.0.sql +./share/postgresql/extension/dblink.control +./share/postgresql/timezone/GB-Eire +./share/postgresql/timezone/Turkey +./share/postgresql/timezone/Kwajalein +./share/postgresql/timezone/UCT +./share/postgresql/timezone/Mexico/BajaSur +./share/postgresql/timezone/Mexico/BajaNorte +./share/postgresql/timezone/Mexico/General +./share/postgresql/timezone/Japan +./share/postgresql/timezone/Israel +./share/postgresql/timezone/US/Eastern +./share/postgresql/timezone/US/Samoa +./share/postgresql/timezone/US/Michigan +./share/postgresql/timezone/US/Aleutian +./share/postgresql/timezone/US/Pacific +./share/postgresql/timezone/US/Pacific-New +./share/postgresql/timezone/US/Indiana-Starke +./share/postgresql/timezone/US/Mountain +./share/postgresql/timezone/US/East-Indiana +./share/postgresql/timezone/US/Hawaii +./share/postgresql/timezone/US/Arizona +./share/postgresql/timezone/US/Alaska +./share/postgresql/timezone/US/Central +./share/postgresql/timezone/Greenwich +./share/postgresql/timezone/Poland +./share/postgresql/timezone/CET +./share/postgresql/timezone/GMT-0 +./share/postgresql/timezone/Indian/Mauritius +./share/postgresql/timezone/Indian/Cocos +./share/postgresql/timezone/Indian/Reunion +./share/postgresql/timezone/Indian/Maldives +./share/postgresql/timezone/Indian/Comoro +./share/postgresql/timezone/Indian/Antananarivo +./share/postgresql/timezone/Indian/Christmas +./share/postgresql/timezone/Indian/Kerguelen +./share/postgresql/timezone/Indian/Chagos +./share/postgresql/timezone/Indian/Mayotte +./share/postgresql/timezone/Indian/Mahe +./share/postgresql/timezone/GMT0 +./share/postgresql/timezone/Antarctica/Palmer +./share/postgresql/timezone/Antarctica/Syowa +./share/postgresql/timezone/Antarctica/South_Pole +./share/postgresql/timezone/Antarctica/McMurdo +./share/postgresql/timezone/Antarctica/Rothera +./share/postgresql/timezone/Antarctica/Mawson +./share/postgresql/timezone/Antarctica/Casey +./share/postgresql/timezone/Antarctica/Davis +./share/postgresql/timezone/Antarctica/DumontDUrville +./share/postgresql/timezone/Antarctica/Vostok +./share/postgresql/timezone/Antarctica/Macquarie +./share/postgresql/timezone/ROK +./share/postgresql/timezone/Chile/EasterIsland +./share/postgresql/timezone/Chile/Continental +./share/postgresql/timezone/posixrules +./share/postgresql/timezone/Atlantic/Azores +./share/postgresql/timezone/Atlantic/St_Helena +./share/postgresql/timezone/Atlantic/Madeira +./share/postgresql/timezone/Atlantic/Jan_Mayen +./share/postgresql/timezone/Atlantic/Faroe +./share/postgresql/timezone/Atlantic/Stanley +./share/postgresql/timezone/Atlantic/Cape_Verde +./share/postgresql/timezone/Atlantic/Reykjavik +./share/postgresql/timezone/Atlantic/Canary +./share/postgresql/timezone/Atlantic/South_Georgia +./share/postgresql/timezone/Atlantic/Bermuda +./share/postgresql/timezone/Atlantic/Faeroe +./share/postgresql/timezone/Hongkong +./share/postgresql/timezone/Libya +./share/postgresql/timezone/Iceland +./share/postgresql/timezone/UTC +./share/postgresql/timezone/Australia/Darwin +./share/postgresql/timezone/Australia/North +./share/postgresql/timezone/Australia/NSW +./share/postgresql/timezone/Australia/Sydney +./share/postgresql/timezone/Australia/Hobart +./share/postgresql/timezone/Australia/LHI +./share/postgresql/timezone/Australia/Victoria +./share/postgresql/timezone/Australia/South +./share/postgresql/timezone/Australia/Melbourne +./share/postgresql/timezone/Australia/Lord_Howe +./share/postgresql/timezone/Australia/West +./share/postgresql/timezone/Australia/Brisbane +./share/postgresql/timezone/Australia/Perth +./share/postgresql/timezone/Australia/Eucla +./share/postgresql/timezone/Australia/Canberra +./share/postgresql/timezone/Australia/Queensland +./share/postgresql/timezone/Australia/Broken_Hill +./share/postgresql/timezone/Australia/Lindeman +./share/postgresql/timezone/Australia/ACT +./share/postgresql/timezone/Australia/Currie +./share/postgresql/timezone/Australia/Adelaide +./share/postgresql/timezone/Australia/Yancowinna +./share/postgresql/timezone/Australia/Tasmania +./share/postgresql/timezone/Jamaica +./share/postgresql/timezone/EST5EDT +./share/postgresql/timezone/MET +./share/postgresql/timezone/W-SU +./share/postgresql/timezone/Mideast/Riyadh87 +./share/postgresql/timezone/Mideast/Riyadh88 +./share/postgresql/timezone/Mideast/Riyadh89 +./share/postgresql/timezone/WET +./share/postgresql/timezone/ROC +./share/postgresql/timezone/Factory +./share/postgresql/timezone/EET +./share/postgresql/timezone/PST8PDT +./share/postgresql/timezone/Portugal +./share/postgresql/timezone/NZ +./share/postgresql/timezone/Brazil/West +./share/postgresql/timezone/Brazil/DeNoronha +./share/postgresql/timezone/Brazil/Acre +./share/postgresql/timezone/Brazil/East +./share/postgresql/timezone/EST +./share/postgresql/timezone/Egypt +./share/postgresql/timezone/Universal +./share/postgresql/timezone/Pacific/Enderbury +./share/postgresql/timezone/Pacific/Noumea +./share/postgresql/timezone/Pacific/Kwajalein +./share/postgresql/timezone/Pacific/Norfolk +./share/postgresql/timezone/Pacific/Nauru +./share/postgresql/timezone/Pacific/Efate +./share/postgresql/timezone/Pacific/Kosrae +./share/postgresql/timezone/Pacific/Galapagos +./share/postgresql/timezone/Pacific/Truk +./share/postgresql/timezone/Pacific/Fiji +./share/postgresql/timezone/Pacific/Auckland +./share/postgresql/timezone/Pacific/Samoa +./share/postgresql/timezone/Pacific/Port_Moresby +./share/postgresql/timezone/Pacific/Johnston +./share/postgresql/timezone/Pacific/Apia +./share/postgresql/timezone/Pacific/Tarawa +./share/postgresql/timezone/Pacific/Pitcairn +./share/postgresql/timezone/Pacific/Marquesas +./share/postgresql/timezone/Pacific/Chatham +./share/postgresql/timezone/Pacific/Tahiti +./share/postgresql/timezone/Pacific/Tongatapu +./share/postgresql/timezone/Pacific/Saipan +./share/postgresql/timezone/Pacific/Fakaofo +./share/postgresql/timezone/Pacific/Guam +./share/postgresql/timezone/Pacific/Niue +./share/postgresql/timezone/Pacific/Chuuk +./share/postgresql/timezone/Pacific/Easter +./share/postgresql/timezone/Pacific/Wallis +./share/postgresql/timezone/Pacific/Gambier +./share/postgresql/timezone/Pacific/Majuro +./share/postgresql/timezone/Pacific/Kiritimati +./share/postgresql/timezone/Pacific/Guadalcanal +./share/postgresql/timezone/Pacific/Funafuti +./share/postgresql/timezone/Pacific/Rarotonga +./share/postgresql/timezone/Pacific/Pago_Pago +./share/postgresql/timezone/Pacific/Midway +./share/postgresql/timezone/Pacific/Palau +./share/postgresql/timezone/Pacific/Honolulu +./share/postgresql/timezone/Pacific/Yap +./share/postgresql/timezone/Pacific/Pohnpei +./share/postgresql/timezone/Pacific/Wake +./share/postgresql/timezone/Pacific/Ponape +./share/postgresql/timezone/Iran +./share/postgresql/timezone/Etc/GMT-4 +./share/postgresql/timezone/Etc/GMT-9 +./share/postgresql/timezone/Etc/UCT +./share/postgresql/timezone/Etc/GMT-7 +./share/postgresql/timezone/Etc/Greenwich +./share/postgresql/timezone/Etc/GMT-0 +./share/postgresql/timezone/Etc/GMT-5 +./share/postgresql/timezone/Etc/GMT0 +./share/postgresql/timezone/Etc/GMT+7 +./share/postgresql/timezone/Etc/GMT-2 +./share/postgresql/timezone/Etc/GMT-10 +./share/postgresql/timezone/Etc/GMT+6 +./share/postgresql/timezone/Etc/GMT+1 +./share/postgresql/timezone/Etc/UTC +./share/postgresql/timezone/Etc/GMT-3 +./share/postgresql/timezone/Etc/GMT-1 +./share/postgresql/timezone/Etc/GMT-8 +./share/postgresql/timezone/Etc/GMT-11 +./share/postgresql/timezone/Etc/GMT+12 +./share/postgresql/timezone/Etc/GMT+10 +./share/postgresql/timezone/Etc/GMT-12 +./share/postgresql/timezone/Etc/GMT+5 +./share/postgresql/timezone/Etc/Universal +./share/postgresql/timezone/Etc/GMT-13 +./share/postgresql/timezone/Etc/GMT-6 +./share/postgresql/timezone/Etc/GMT+9 +./share/postgresql/timezone/Etc/GMT+3 +./share/postgresql/timezone/Etc/GMT-14 +./share/postgresql/timezone/Etc/GMT+4 +./share/postgresql/timezone/Etc/Zulu +./share/postgresql/timezone/Etc/GMT+2 +./share/postgresql/timezone/Etc/GMT+0 +./share/postgresql/timezone/Etc/GMT+8 +./share/postgresql/timezone/Etc/GMT+11 +./share/postgresql/timezone/Etc/GMT +./share/postgresql/timezone/Zulu +./share/postgresql/timezone/GMT+0 +./share/postgresql/timezone/Singapore +./share/postgresql/timezone/NZ-CHAT +./share/postgresql/timezone/Cuba +./share/postgresql/timezone/GB +./share/postgresql/timezone/Arctic/Longyearbyen +./share/postgresql/timezone/MST7MDT +./share/postgresql/timezone/PRC +./share/postgresql/timezone/Canada/Eastern +./share/postgresql/timezone/Canada/Yukon +./share/postgresql/timezone/Canada/Atlantic +./share/postgresql/timezone/Canada/Newfoundland +./share/postgresql/timezone/Canada/Saskatchewan +./share/postgresql/timezone/Canada/Pacific +./share/postgresql/timezone/Canada/East-Saskatchewan +./share/postgresql/timezone/Canada/Mountain +./share/postgresql/timezone/Canada/Central +./share/postgresql/timezone/CST6CDT +./share/postgresql/timezone/HST +./share/postgresql/timezone/America/Boa_Vista +./share/postgresql/timezone/America/New_York +./share/postgresql/timezone/America/Santarem +./share/postgresql/timezone/America/Boise +./share/postgresql/timezone/America/St_Lucia +./share/postgresql/timezone/America/Mendoza +./share/postgresql/timezone/America/Mexico_City +./share/postgresql/timezone/America/Chihuahua +./share/postgresql/timezone/America/Indianapolis +./share/postgresql/timezone/America/Virgin +./share/postgresql/timezone/America/Atka +./share/postgresql/timezone/America/Winnipeg +./share/postgresql/timezone/America/Hermosillo +./share/postgresql/timezone/America/Indiana/Indianapolis +./share/postgresql/timezone/America/Indiana/Tell_City +./share/postgresql/timezone/America/Indiana/Winamac +./share/postgresql/timezone/America/Indiana/Knox +./share/postgresql/timezone/America/Indiana/Vincennes +./share/postgresql/timezone/America/Indiana/Vevay +./share/postgresql/timezone/America/Indiana/Petersburg +./share/postgresql/timezone/America/Indiana/Marengo +./share/postgresql/timezone/America/Moncton +./share/postgresql/timezone/America/Campo_Grande +./share/postgresql/timezone/America/Guyana +./share/postgresql/timezone/America/Caracas +./share/postgresql/timezone/America/Maceio +./share/postgresql/timezone/America/Godthab +./share/postgresql/timezone/America/Thunder_Bay +./share/postgresql/timezone/America/Havana +./share/postgresql/timezone/America/Santiago +./share/postgresql/timezone/America/Los_Angeles +./share/postgresql/timezone/America/Buenos_Aires +./share/postgresql/timezone/America/Manaus +./share/postgresql/timezone/America/Bahia +./share/postgresql/timezone/America/North_Dakota/New_Salem +./share/postgresql/timezone/America/North_Dakota/Beulah +./share/postgresql/timezone/America/North_Dakota/Center +./share/postgresql/timezone/America/Bahia_Banderas +./share/postgresql/timezone/America/Edmonton +./share/postgresql/timezone/America/Tegucigalpa +./share/postgresql/timezone/America/Rankin_Inlet +./share/postgresql/timezone/America/Monterrey +./share/postgresql/timezone/America/Cambridge_Bay +./share/postgresql/timezone/America/Porto_Velho +./share/postgresql/timezone/America/Antigua +./share/postgresql/timezone/America/Atikokan +./share/postgresql/timezone/America/Vancouver +./share/postgresql/timezone/America/Anchorage +./share/postgresql/timezone/America/Port-au-Prince +./share/postgresql/timezone/America/Lima +./share/postgresql/timezone/America/Grenada +./share/postgresql/timezone/America/Creston +./share/postgresql/timezone/America/La_Paz +./share/postgresql/timezone/America/Panama +./share/postgresql/timezone/America/Blanc-Sablon +./share/postgresql/timezone/America/Cayenne +./share/postgresql/timezone/America/Santo_Domingo +./share/postgresql/timezone/America/Grand_Turk +./share/postgresql/timezone/America/Toronto +./share/postgresql/timezone/America/Rainy_River +./share/postgresql/timezone/America/Merida +./share/postgresql/timezone/America/Port_of_Spain +./share/postgresql/timezone/America/Nipigon +./share/postgresql/timezone/America/Jamaica +./share/postgresql/timezone/America/Rosario +./share/postgresql/timezone/America/Dawson_Creek +./share/postgresql/timezone/America/Belize +./share/postgresql/timezone/America/Costa_Rica +./share/postgresql/timezone/America/Barbados +./share/postgresql/timezone/America/Danmarkshavn +./share/postgresql/timezone/America/Argentina/La_Rioja +./share/postgresql/timezone/America/Argentina/Mendoza +./share/postgresql/timezone/America/Argentina/Buenos_Aires +./share/postgresql/timezone/America/Argentina/Tucuman +./share/postgresql/timezone/America/Argentina/Ushuaia +./share/postgresql/timezone/America/Argentina/Catamarca +./share/postgresql/timezone/America/Argentina/ComodRivadavia +./share/postgresql/timezone/America/Argentina/Jujuy +./share/postgresql/timezone/America/Argentina/Cordoba +./share/postgresql/timezone/America/Argentina/San_Luis +./share/postgresql/timezone/America/Argentina/Rio_Gallegos +./share/postgresql/timezone/America/Argentina/Salta +./share/postgresql/timezone/America/Argentina/San_Juan +./share/postgresql/timezone/America/Pangnirtung +./share/postgresql/timezone/America/Anguilla +./share/postgresql/timezone/America/Curacao +./share/postgresql/timezone/America/Cancun +./share/postgresql/timezone/America/Montreal +./share/postgresql/timezone/America/Shiprock +./share/postgresql/timezone/America/Thule +./share/postgresql/timezone/America/Scoresbysund +./share/postgresql/timezone/America/Catamarca +./share/postgresql/timezone/America/Sao_Paulo +./share/postgresql/timezone/America/Sitka +./share/postgresql/timezone/America/Asuncion +./share/postgresql/timezone/America/Regina +./share/postgresql/timezone/America/St_Johns +./share/postgresql/timezone/America/Montevideo +./share/postgresql/timezone/America/Eirunepe +./share/postgresql/timezone/America/Denver +./share/postgresql/timezone/America/Metlakatla +./share/postgresql/timezone/America/Araguaina +./share/postgresql/timezone/America/Juneau +./share/postgresql/timezone/America/Marigot +./share/postgresql/timezone/America/Menominee +./share/postgresql/timezone/America/Glace_Bay +./share/postgresql/timezone/America/Tijuana +./share/postgresql/timezone/America/Detroit +./share/postgresql/timezone/America/Belem +./share/postgresql/timezone/America/Jujuy +./share/postgresql/timezone/America/St_Thomas +./share/postgresql/timezone/America/Resolute +./share/postgresql/timezone/America/Cuiaba +./share/postgresql/timezone/America/Halifax +./share/postgresql/timezone/America/St_Barthelemy +./share/postgresql/timezone/America/Guatemala +./share/postgresql/timezone/America/Nassau +./share/postgresql/timezone/America/St_Kitts +./share/postgresql/timezone/America/Cordoba +./share/postgresql/timezone/America/Miquelon +./share/postgresql/timezone/America/Bogota +./share/postgresql/timezone/America/Rio_Branco +./share/postgresql/timezone/America/Ensenada +./share/postgresql/timezone/America/Yakutat +./share/postgresql/timezone/America/Noronha +./share/postgresql/timezone/America/Kentucky/Monticello +./share/postgresql/timezone/America/Kentucky/Louisville +./share/postgresql/timezone/America/Porto_Acre +./share/postgresql/timezone/America/Santa_Isabel +./share/postgresql/timezone/America/El_Salvador +./share/postgresql/timezone/America/Yellowknife +./share/postgresql/timezone/America/Cayman +./share/postgresql/timezone/America/Whitehorse +./share/postgresql/timezone/America/Ojinaga +./share/postgresql/timezone/America/Aruba +./share/postgresql/timezone/America/Nome +./share/postgresql/timezone/America/Fortaleza +./share/postgresql/timezone/America/Martinique +./share/postgresql/timezone/America/Recife +./share/postgresql/timezone/America/Knox_IN +./share/postgresql/timezone/America/Guayaquil +./share/postgresql/timezone/America/Goose_Bay +./share/postgresql/timezone/America/Iqaluit +./share/postgresql/timezone/America/Matamoros +./share/postgresql/timezone/America/Lower_Princes +./share/postgresql/timezone/America/Louisville +./share/postgresql/timezone/America/Coral_Harbour +./share/postgresql/timezone/America/Phoenix +./share/postgresql/timezone/America/Guadeloupe +./share/postgresql/timezone/America/Mazatlan +./share/postgresql/timezone/America/Swift_Current +./share/postgresql/timezone/America/Paramaribo +./share/postgresql/timezone/America/Dominica +./share/postgresql/timezone/America/Kralendijk +./share/postgresql/timezone/America/Montserrat +./share/postgresql/timezone/America/St_Vincent +./share/postgresql/timezone/America/Fort_Wayne +./share/postgresql/timezone/America/Dawson +./share/postgresql/timezone/America/Inuvik +./share/postgresql/timezone/America/Adak +./share/postgresql/timezone/America/Managua +./share/postgresql/timezone/America/Puerto_Rico +./share/postgresql/timezone/America/Tortola +./share/postgresql/timezone/America/Chicago +./share/postgresql/timezone/Africa/Lome +./share/postgresql/timezone/Africa/Brazzaville +./share/postgresql/timezone/Africa/Khartoum +./share/postgresql/timezone/Africa/Ceuta +./share/postgresql/timezone/Africa/Djibouti +./share/postgresql/timezone/Africa/Lagos +./share/postgresql/timezone/Africa/Accra +./share/postgresql/timezone/Africa/El_Aaiun +./share/postgresql/timezone/Africa/Malabo +./share/postgresql/timezone/Africa/Windhoek +./share/postgresql/timezone/Africa/Tripoli +./share/postgresql/timezone/Africa/Bissau +./share/postgresql/timezone/Africa/Blantyre +./share/postgresql/timezone/Africa/Kinshasa +./share/postgresql/timezone/Africa/Porto-Novo +./share/postgresql/timezone/Africa/Nairobi +./share/postgresql/timezone/Africa/Ouagadougou +./share/postgresql/timezone/Africa/Asmera +./share/postgresql/timezone/Africa/Cairo +./share/postgresql/timezone/Africa/Lubumbashi +./share/postgresql/timezone/Africa/Tunis +./share/postgresql/timezone/Africa/Dar_es_Salaam +./share/postgresql/timezone/Africa/Casablanca +./share/postgresql/timezone/Africa/Algiers +./share/postgresql/timezone/Africa/Mbabane +./share/postgresql/timezone/Africa/Monrovia +./share/postgresql/timezone/Africa/Nouakchott +./share/postgresql/timezone/Africa/Banjul +./share/postgresql/timezone/Africa/Kampala +./share/postgresql/timezone/Africa/Conakry +./share/postgresql/timezone/Africa/Mogadishu +./share/postgresql/timezone/Africa/Ndjamena +./share/postgresql/timezone/Africa/Niamey +./share/postgresql/timezone/Africa/Lusaka +./share/postgresql/timezone/Africa/Addis_Ababa +./share/postgresql/timezone/Africa/Sao_Tome +./share/postgresql/timezone/Africa/Abidjan +./share/postgresql/timezone/Africa/Harare +./share/postgresql/timezone/Africa/Asmara +./share/postgresql/timezone/Africa/Douala +./share/postgresql/timezone/Africa/Freetown +./share/postgresql/timezone/Africa/Libreville +./share/postgresql/timezone/Africa/Luanda +./share/postgresql/timezone/Africa/Maseru +./share/postgresql/timezone/Africa/Gaborone +./share/postgresql/timezone/Africa/Maputo +./share/postgresql/timezone/Africa/Timbuktu +./share/postgresql/timezone/Africa/Bangui +./share/postgresql/timezone/Africa/Bamako +./share/postgresql/timezone/Africa/Dakar +./share/postgresql/timezone/Africa/Juba +./share/postgresql/timezone/Africa/Bujumbura +./share/postgresql/timezone/Africa/Johannesburg +./share/postgresql/timezone/Africa/Kigali +./share/postgresql/timezone/Eire +./share/postgresql/timezone/Europe/Vaduz +./share/postgresql/timezone/Europe/Podgorica +./share/postgresql/timezone/Europe/Rome +./share/postgresql/timezone/Europe/Vienna +./share/postgresql/timezone/Europe/Dublin +./share/postgresql/timezone/Europe/Zurich +./share/postgresql/timezone/Europe/London +./share/postgresql/timezone/Europe/Monaco +./share/postgresql/timezone/Europe/Sofia +./share/postgresql/timezone/Europe/Uzhgorod +./share/postgresql/timezone/Europe/Minsk +./share/postgresql/timezone/Europe/Malta +./share/postgresql/timezone/Europe/Busingen +./share/postgresql/timezone/Europe/Gibraltar +./share/postgresql/timezone/Europe/Volgograd +./share/postgresql/timezone/Europe/Budapest +./share/postgresql/timezone/Europe/Vatican +./share/postgresql/timezone/Europe/Luxembourg +./share/postgresql/timezone/Europe/Chisinau +./share/postgresql/timezone/Europe/Nicosia +./share/postgresql/timezone/Europe/Warsaw +./share/postgresql/timezone/Europe/San_Marino +./share/postgresql/timezone/Europe/Copenhagen +./share/postgresql/timezone/Europe/Ljubljana +./share/postgresql/timezone/Europe/Athens +./share/postgresql/timezone/Europe/Skopje +./share/postgresql/timezone/Europe/Andorra +./share/postgresql/timezone/Europe/Kaliningrad +./share/postgresql/timezone/Europe/Amsterdam +./share/postgresql/timezone/Europe/Guernsey +./share/postgresql/timezone/Europe/Isle_of_Man +./share/postgresql/timezone/Europe/Tirane +./share/postgresql/timezone/Europe/Jersey +./share/postgresql/timezone/Europe/Madrid +./share/postgresql/timezone/Europe/Helsinki +./share/postgresql/timezone/Europe/Riga +./share/postgresql/timezone/Europe/Zagreb +./share/postgresql/timezone/Europe/Bratislava +./share/postgresql/timezone/Europe/Prague +./share/postgresql/timezone/Europe/Tallinn +./share/postgresql/timezone/Europe/Stockholm +./share/postgresql/timezone/Europe/Tiraspol +./share/postgresql/timezone/Europe/Belgrade +./share/postgresql/timezone/Europe/Bucharest +./share/postgresql/timezone/Europe/Vilnius +./share/postgresql/timezone/Europe/Sarajevo +./share/postgresql/timezone/Europe/Belfast +./share/postgresql/timezone/Europe/Zaporozhye +./share/postgresql/timezone/Europe/Oslo +./share/postgresql/timezone/Europe/Mariehamn +./share/postgresql/timezone/Europe/Moscow +./share/postgresql/timezone/Europe/Brussels +./share/postgresql/timezone/Europe/Paris +./share/postgresql/timezone/Europe/Istanbul +./share/postgresql/timezone/Europe/Simferopol +./share/postgresql/timezone/Europe/Lisbon +./share/postgresql/timezone/Europe/Berlin +./share/postgresql/timezone/Europe/Kiev +./share/postgresql/timezone/Europe/Samara +./share/postgresql/timezone/MST +./share/postgresql/timezone/Asia/Khandyga +./share/postgresql/timezone/Asia/Manila +./share/postgresql/timezone/Asia/Novokuznetsk +./share/postgresql/timezone/Asia/Baghdad +./share/postgresql/timezone/Asia/Macau +./share/postgresql/timezone/Asia/Urumqi +./share/postgresql/timezone/Asia/Ujung_Pandang +./share/postgresql/timezone/Asia/Ulan_Bator +./share/postgresql/timezone/Asia/Bishkek +./share/postgresql/timezone/Asia/Qatar +./share/postgresql/timezone/Asia/Qyzylorda +./share/postgresql/timezone/Asia/Calcutta +./share/postgresql/timezone/Asia/Riyadh87 +./share/postgresql/timezone/Asia/Dushanbe +./share/postgresql/timezone/Asia/Yekaterinburg +./share/postgresql/timezone/Asia/Dhaka +./share/postgresql/timezone/Asia/Jakarta +./share/postgresql/timezone/Asia/Shanghai +./share/postgresql/timezone/Asia/Ulaanbaatar +./share/postgresql/timezone/Asia/Jerusalem +./share/postgresql/timezone/Asia/Ashkhabad +./share/postgresql/timezone/Asia/Tokyo +./share/postgresql/timezone/Asia/Macao +./share/postgresql/timezone/Asia/Krasnoyarsk +./share/postgresql/timezone/Asia/Saigon +./share/postgresql/timezone/Asia/Omsk +./share/postgresql/timezone/Asia/Damascus +./share/postgresql/timezone/Asia/Phnom_Penh +./share/postgresql/timezone/Asia/Bangkok +./share/postgresql/timezone/Asia/Kamchatka +./share/postgresql/timezone/Asia/Choibalsan +./share/postgresql/timezone/Asia/Ust-Nera +./share/postgresql/timezone/Asia/Aden +./share/postgresql/timezone/Asia/Vientiane +./share/postgresql/timezone/Asia/Sakhalin +./share/postgresql/timezone/Asia/Ashgabat +./share/postgresql/timezone/Asia/Katmandu +./share/postgresql/timezone/Asia/Almaty +./share/postgresql/timezone/Asia/Baku +./share/postgresql/timezone/Asia/Nicosia +./share/postgresql/timezone/Asia/Riyadh88 +./share/postgresql/timezone/Asia/Kashgar +./share/postgresql/timezone/Asia/Riyadh89 +./share/postgresql/timezone/Asia/Taipei +./share/postgresql/timezone/Asia/Tehran +./share/postgresql/timezone/Asia/Kabul +./share/postgresql/timezone/Asia/Samarkand +./share/postgresql/timezone/Asia/Kuala_Lumpur +./share/postgresql/timezone/Asia/Tashkent +./share/postgresql/timezone/Asia/Thimbu +./share/postgresql/timezone/Asia/Thimphu +./share/postgresql/timezone/Asia/Yerevan +./share/postgresql/timezone/Asia/Chungking +./share/postgresql/timezone/Asia/Hebron +./share/postgresql/timezone/Asia/Karachi +./share/postgresql/timezone/Asia/Kolkata +./share/postgresql/timezone/Asia/Aqtobe +./share/postgresql/timezone/Asia/Muscat +./share/postgresql/timezone/Asia/Hong_Kong +./share/postgresql/timezone/Asia/Chongqing +./share/postgresql/timezone/Asia/Oral +./share/postgresql/timezone/Asia/Pontianak +./share/postgresql/timezone/Asia/Colombo +./share/postgresql/timezone/Asia/Pyongyang +./share/postgresql/timezone/Asia/Hovd +./share/postgresql/timezone/Asia/Kuwait +./share/postgresql/timezone/Asia/Anadyr +./share/postgresql/timezone/Asia/Kathmandu +./share/postgresql/timezone/Asia/Irkutsk +./share/postgresql/timezone/Asia/Bahrain +./share/postgresql/timezone/Asia/Dubai +./share/postgresql/timezone/Asia/Jayapura +./share/postgresql/timezone/Asia/Riyadh +./share/postgresql/timezone/Asia/Ho_Chi_Minh +./share/postgresql/timezone/Asia/Singapore +./share/postgresql/timezone/Asia/Tel_Aviv +./share/postgresql/timezone/Asia/Dili +./share/postgresql/timezone/Asia/Rangoon +./share/postgresql/timezone/Asia/Harbin +./share/postgresql/timezone/Asia/Yakutsk +./share/postgresql/timezone/Asia/Magadan +./share/postgresql/timezone/Asia/Amman +./share/postgresql/timezone/Asia/Kuching +./share/postgresql/timezone/Asia/Novosibirsk +./share/postgresql/timezone/Asia/Seoul +./share/postgresql/timezone/Asia/Dacca +./share/postgresql/timezone/Asia/Vladivostok +./share/postgresql/timezone/Asia/Istanbul +./share/postgresql/timezone/Asia/Beirut +./share/postgresql/timezone/Asia/Aqtau +./share/postgresql/timezone/Asia/Brunei +./share/postgresql/timezone/Asia/Gaza +./share/postgresql/timezone/Asia/Tbilisi +./share/postgresql/timezone/Asia/Makassar +./share/postgresql/timezone/Asia/Beijing +./share/postgresql/timezone/Navajo +./share/postgresql/timezone/GMT +./share/postgresql/system_views.sql +./share/postgresql/private_system_views.sql +./share/postgresql/performance_views.sql +./share/postgresql/sql_features.txt +./share/postgresql/pg_cast_oid.txt +./share/postgresql/recovery.conf.sample +./share/postgresql/tsearch_data/english.stop +./share/postgresql/tsearch_data/dutch.stop +./share/postgresql/tsearch_data/hungarian.stop +./share/postgresql/tsearch_data/french.stop +./share/postgresql/tsearch_data/synonym_sample.syn +./share/postgresql/tsearch_data/turkish.stop +./share/postgresql/tsearch_data/portuguese.stop +./share/postgresql/tsearch_data/spanish.stop +./share/postgresql/tsearch_data/hunspell_sample.affix +./share/postgresql/tsearch_data/ispell_sample.affix +./share/postgresql/tsearch_data/danish.stop +./share/postgresql/tsearch_data/german.stop +./share/postgresql/tsearch_data/thesaurus_sample.ths +./share/postgresql/tsearch_data/norwegian.stop +./share/postgresql/tsearch_data/finnish.stop +./share/postgresql/tsearch_data/russian.stop +./share/postgresql/tsearch_data/swedish.stop +./share/postgresql/tsearch_data/ispell_sample.dict +./share/postgresql/tsearch_data/italian.stop +./share/postgresql/information_schema.sql +./share/postgresql/timezonesets/Antarctica.txt +./share/postgresql/timezonesets/Australia.txt +./share/postgresql/timezonesets/Europe.txt +./share/postgresql/timezonesets/America.txt +./share/postgresql/timezonesets/Australia +./share/postgresql/timezonesets/Indian.txt +./share/postgresql/timezonesets/India +./share/postgresql/timezonesets/Pacific.txt +./share/postgresql/timezonesets/Atlantic.txt +./share/postgresql/timezonesets/Africa.txt +./share/postgresql/timezonesets/Asia.txt +./share/postgresql/timezonesets/Default +./share/postgresql/timezonesets/Etc.txt +./share/postgresql/postgres.bki +./share/llvmir/GaussDB_expr.ir +./share/sslcert/gsql/openssl.cnf +./share/sslcert/grpc/openssl.cnf +./share/sslcert/om/openssl.cnf +./lib/libsimsearch/ +./lib/postgresql/latin2_and_win1250.so +./lib/postgresql/euc2004_sjis2004.so +./lib/postgresql/euc_kr_and_mic.so +./lib/postgresql/utf8_and_uhc.so +./lib/postgresql/euc_tw_and_big5.so +./lib/postgresql/cyrillic_and_mic.so +./lib/postgresql/utf8_and_johab.so +./lib/postgresql/utf8_and_gb18030.so +./lib/postgresql/pgxs/src/makefiles/pgxs.mk +./lib/postgresql/pgxs/src/Makefile.shlib +./lib/postgresql/pgxs/src/Makefile.port +./lib/postgresql/pgxs/src/nls-global.mk +./lib/postgresql/pgxs/src/Makefile.global +./lib/postgresql/pgxs/config/install-sh +./lib/postgresql/euc_cn_and_mic.so +./lib/postgresql/latin_and_mic.so +./lib/postgresql/utf8_and_sjis2004.so +./lib/postgresql/utf8_and_euc_jp.so +./lib/postgresql/utf8_and_sjis.so +./lib/postgresql/utf8_and_cyrillic.so +./lib/postgresql/hstore.so +./lib/postgresql/utf8_and_euc_kr.so +./lib/postgresql/ascii_and_mic.so +./lib/postgresql/utf8_and_iso8859_1.so +./lib/postgresql/euc_jp_and_sjis.so +./lib/postgresql/dict_snowball.so +./lib/postgresql/utf8_and_ascii.so +./lib/postgresql/utf8_and_euc_tw.so +./lib/postgresql/utf8_and_iso8859.so +./lib/postgresql/utf8_and_win.so +./lib/postgresql/utf8_and_euc_cn.so +./lib/postgresql/utf8_and_gbk.so +./lib/postgresql/utf8_and_euc2004.so +./lib/postgresql/utf8_and_big5.so +./lib/postgresql/mppdb_decoding.so +./lib/postgresql/ndpplugin.so +./lib/postgresql/pg_plugin +./lib/postgresql/proc_srclib +./lib/postgresql/security_plugin.so +./lib/postgresql/dolphin.so +./lib/postgresql/pg_upgrade_support.so +./lib/postgresql/java/pljava.jar +./lib/postgresql/postgres_fdw.so +./lib/postgresql/dblink.so +./lib/postgresql/pgoutput.so +./lib/postgresql/assessment.so +./lib/libpljava.so +./lib/libpq.a +./lib/libpq.so +./lib/libpq.so.5 +./lib/libpq.so.5.5 +./lib/libpq_ce.so +./lib/libpq_ce.so.5 +./lib/libpq_ce.so.5.5 +./lib/libgauss_cl_jni.so +./lib/libnuma.so +./lib/libnuma.so.1 +./lib/libnuma.so.1.0.0 +./lib/libcgroup.so* +./lib/libcom_err_gauss.so* +./lib/libatomic.so* +./lib/libmasstree.so +./lib/libupb.so* +./lib/libabsl_str_format_internal.so +./lib/libabsl_strings.so +./lib/libabsl_throw_delegate.so +./lib/libabsl_strings_internal.so +./lib/libabsl_base.so +./lib/libabsl_dynamic_annotations.so +./lib/libabsl_spinlock_wait.so +./lib/libabsl_int128.so +./lib/libabsl_bad_optional_access.so +./lib/libabsl_raw_logging_internal.so +./lib/libabsl_log_severity.so +./lib/libaddress_sorting.so +./lib/libaddress_sorting.so.9 +./lib/libgssapi_krb5_gauss.so* +./lib/libgssrpc_gauss.so* +./lib/libk5crypto_gauss.so* +./lib/libkadm5clnt.so +./lib/libkadm5clnt_mit.so* +./lib/libkadm5srv.so +./lib/libkadm5srv_mit.so* +./lib/libkdb5.so* +./lib/libkrad.so* +./lib/libkrb5_gauss.so* +./lib/libkrb5support_gauss.so* +./lib/krb5/plugins/kdb/db2.so +./lib/libverto.so* +./lib/libcurl.so* +./lib/libcrypto.so* +./lib/libssl.so* +./lib/libgcc_s.so.1 +./lib/libstdc++.so.6 +./lib/libz.so* +./lib/liblz4.so* +./lib/libcjson.so* +./lib/libconfig.so* +./lib/libpgport_tool.so +./lib/libpgport_tool.so.1 +./share/llvmir/GaussDB_expr.ir +./lib/libeSDKLogAPI.so +./lib/libeSDKOBS.so +./lib/liblog4cpp.so* +./lib/libcharset.so* +./lib/libiconv.so* +./lib/libnghttp2.so* +./lib/libpcre.so* +./lib/libsecurec.so +./lib/libxml2.so* +./lib/OBS.ini +./lib/postgresql/latin2_and_win1250.so +./lib/postgresql/euc2004_sjis2004.so +./lib/libdcf.so +./lib/libzstd.so* +./lib/libxgboost.so +./lib/libpagecompression.so* +./lib/libdssapi.so +./lib/libdms.so +./lib/libodbc.so* + +./include/postgresql/server/postgres_ext.h +./include/postgresql/server/pg_config_os.h +./include/postgresql/server/pgtime.h +./include/postgresql/server/datatypes.h +./include/postgresql/server/client_logic/client_logic_enums.h +./include/postgresql/server/nodes/primnodes.h +./include/postgresql/server/nodes/parsenodes.h +./include/postgresql/server/nodes/parsenodes_common.h +./include/postgresql/server/nodes/bitmapset.h +./include/postgresql/server/nodes/pg_list.h +./include/postgresql/server/nodes/value.h +./include/postgresql/server/nodes/nodes.h +./include/postgresql/server/utils/sortsupport.h +./include/postgresql/server/utils/varbit.h +./include/postgresql/server/utils/spccache.h +./include/postgresql/server/utils/rangetypes.h +./include/postgresql/server/utils/plpgsql.h +./include/postgresql/server/utils/memtrack.h +./include/postgresql/server/utils/pg_locale.h +./include/postgresql/server/utils/tzparser.h +./include/postgresql/server/utils/syscall_lock.h +./include/postgresql/server/utils/partitionmap.h +./include/postgresql/server/utils/array.h +./include/postgresql/server/utils/relmapper.h +./include/postgresql/server/utils/hsearch.h +./include/postgresql/server/utils/xml.h +./include/postgresql/server/utils/bytea.h +./include/postgresql/server/utils/relcache.h +./include/postgresql/server/utils/pg_rusage.h +./include/postgresql/server/utils/numeric.h +./include/postgresql/server/utils/mmpool.h +./include/postgresql/server/utils/nabstime.h +./include/postgresql/server/utils/fmgrtab.h +./include/postgresql/server/utils/snapmgr.h +./include/postgresql/server/utils/syscache.h +./include/postgresql/server/utils/logtape.h +./include/postgresql/server/utils/datum.h +./include/postgresql/server/utils/guc_tables.h +./include/postgresql/server/utils/snapshot.h +./include/postgresql/server/utils/geo_decls.h +./include/postgresql/server/utils/errcodes.h +./include/postgresql/server/utils/inval.h +./include/postgresql/server/utils/help_config.h +./include/postgresql/server/utils/distribute_test.h +./include/postgresql/server/utils/aiomem.h +./include/postgresql/server/utils/tuplestore.h +./include/postgresql/server/utils/rbtree.h +./include/postgresql/server/utils/gs_bitmap.h +./include/postgresql/server/utils/tuplesort.h +./include/postgresql/server/utils/ps_status.h +./include/postgresql/server/utils/palloc.h +./include/postgresql/server/utils/reltrigger.h +./include/postgresql/server/utils/acl.h +./include/postgresql/server/utils/ascii.h +./include/postgresql/server/utils/selfuncs.h +./include/postgresql/server/utils/json.h +./include/postgresql/server/utils/portal.h +./include/postgresql/server/utils/atomic.h +./include/postgresql/server/utils/elog.h +./include/postgresql/server/utils/date.h +./include/postgresql/server/utils/plancache.h +./include/postgresql/server/utils/int8.h +./include/postgresql/server/utils/timestamp.h +./include/postgresql/server/utils/bloom_filter.h +./include/postgresql/server/utils/fmgroids.h +./include/postgresql/server/utils/pg_crc_tables.h +./include/postgresql/server/utils/probes.h +./include/postgresql/server/utils/datetime.h +./include/postgresql/server/utils/inet.h +./include/postgresql/server/utils/pg_lzcompress.h +./include/postgresql/server/utils/pg_crc.h +./include/postgresql/server/utils/attoptcache.h +./include/postgresql/server/utils/dynahash.h +./include/postgresql/server/utils/rel.h +./include/postgresql/server/utils/partcache.h +./include/postgresql/server/utils/lsyscache.h +./include/postgresql/server/utils/memutils.h +./include/postgresql/server/utils/memprot.h +./include/postgresql/server/utils/uuid.h +./include/postgresql/server/utils/combocid.h +./include/postgresql/server/utils/builtins.h +./include/postgresql/server/utils/guc.h +./include/postgresql/server/utils/dfs_vector.h +./include/postgresql/server/utils/dynamic_loader.h +./include/postgresql/server/utils/resowner.h +./include/postgresql/server/utils/aes.h +./include/postgresql/server/utils/cash.h +./include/postgresql/server/utils/typcache.h +./include/postgresql/server/utils/formatting.h +./include/postgresql/server/utils/partitionkey.h +./include/postgresql/server/utils/aset.h +./include/postgresql/server/utils/catcache.h +./include/postgresql/server/utils/atomic_arm.h +./include/postgresql/server/utils/oidrbtree.h +./include/postgresql/server/datatype/timestamp.h +./include/postgresql/server/access/rmgr.h +./include/postgresql/server/access/xlogreader.h +./include/postgresql/server/access/xlog_basic.h +./include/postgresql/server/access/tupdesc.h +./include/postgresql/server/access/rmgrlist.h +./include/postgresql/server/access/htup.h +./include/postgresql/server/access/xlogdefs.h +./include/postgresql/server/access/attnum.h +./include/postgresql/server/access/tupmacs.h +./include/postgresql/server/access/xlogrecord.h +./include/postgresql/server/tde_key_management/data_common.h +./include/postgresql/server/tcop/dest.h +./include/postgresql/server/catalog/pg_type.h +./include/postgresql/server/catalog/pg_attribute.h +./include/postgresql/server/catalog/genbki.h +./include/postgresql/server/gs_thread.h +./include/postgresql/server/port/pg_bswap.h +./include/postgresql/server/port/pg_crc32c.h +./include/postgresql/server/securec.h +./include/postgresql/server/securectype.h +./include/postgresql/server/storage/off.h +./include/postgresql/server/storage/buf/block.h +./include/postgresql/server/storage/item/item.h +./include/postgresql/server/storage/smgr/relfilenode.h +./include/postgresql/server/storage/buf/bufpage.h +./include/postgresql/server/storage/spin.h +./include/postgresql/server/storage/buf/buf.h +./include/postgresql/server/storage/item/itemid.h +./include/postgresql/server/storage/lock/pg_sema.h +./include/postgresql/server/storage/item/itemptr.h +./include/postgresql/server/storage/lock/s_lock.h +./include/postgresql/server/storage/backendid.h +./include/postgresql/server/storage/lock/lock.h +./include/postgresql/server/storage/lock/lwlock.h +./include/postgresql/server/storage/lwlocknames.h +./include/postgresql/server/storage/barrier.h +./include/postgresql/server/storage/shmem.h +./include/postgresql/server/pg_config.h +./include/postgresql/server/lib/stringinfo.h +./include/postgresql/server/fmgr.h +./include/postgresql/server/fmgr/fmgr_comp.h +./include/postgresql/server/fmgr/fmgr_core.h +./include/postgresql/server/gs_threadlocal.h +./include/postgresql/server/postgres.h +./include/postgresql/server/executor/tuptable.h +./include/postgresql/server/pg_config_manual.h +./include/postgresql/server/mb/pg_wchar.h +./include/postgresql/server/c.h +./include/postgresql/server/port.h +./include/postgresql/server/utils/be_module.h +./include/postgresql/server/nodes/params.h +./include/postgresql/server/securec_check.h +./include/postgresql/server/nodes/memnodes.h +./include/postgresql/server/access/skey.h +./include/postgresql/server/lib/dllist.h +./include/postgresql/server/lib/ilist.h +./include/postgresql/server/pgxc/locator.h +./include/postgresql/server/gstrace/gstrace_infra.h +./include/postgresql/server/extension_dependency.h +./include/postgresql/server/libpq/libpq-fe.h +./include/postgresql/server/access/clog.h +./include/postgresql/server/storage/proc.h +./include/postgresql/server/access/xlog.h +./include/postgresql/server/storage/lwlocknames.h +./include/postgresql/server/access/xloginsert.h +./include/postgresql/server/catalog/pg_control.h +./include/postgresql/server/access/parallel_recovery/redo_item.h +./include/postgresql/server/access/parallel_recovery/posix_semaphore.h +./include/postgresql/server/replication/replicainternal.h +./include/postgresql/server/knl/knl_instance.h +./include/postgresql/server/knl/knl_guc.h +./include/postgresql/server/knl/knl_guc/knl_session_attr_sql.h +./include/postgresql/server/knl/knl_guc/knl_guc_common.h +./include/postgresql/server/knl/knl_guc/knl_instance_attr_sql.h +./include/postgresql/server/knl/knl_guc/knl_session_attr_storage.h +./include/postgresql/server/knl/knl_guc/knl_instance_attr_storage.h +./include/postgresql/server/knl/knl_guc/knl_instance_attr_storage.h +./include/postgresql/server/knl/knl_guc/knl_session_attr_security.h +./include/postgresql/server/knl/knl_guc/knl_instance_attr_security.h +./include/postgresql/server/knl/knl_guc/knl_session_attr_network.h +./include/postgresql/server/knl/knl_guc/knl_instance_attr_network.h +./include/postgresql/server/knl/knl_guc/knl_session_attr_memory.h +./include/postgresql/server/knl/knl_guc/knl_instance_attr_memory.h +./include/postgresql/server/knl/knl_guc/knl_session_attr_resource.h +./include/postgresql/server/knl/knl_guc/knl_instance_attr_resource.h +./include/postgresql/server/knl/knl_guc/knl_session_attr_common.h +./include/postgresql/server/knl/knl_guc/knl_instance_attr_common.h +./include/postgresql/server/access/ustore/knl_uverify.h +./include/postgresql/server/lib/circularqueue.h +./include/postgresql/server/access/double_write_basic.h +./include/postgresql/server/knl/knl_thread.h +./include/postgresql/server/access/sdir.h +./include/postgresql/server/gssignal/gs_signal.h +./include/postgresql/server/knl/knl_session.h +./include/postgresql/server/libpq/pqcomm.h +./include/postgresql/server/cipher.h +./include/postgresql/server/portability/instr_time.h +./include/postgresql/server/utils/memgroup.h +./include/postgresql/server/storage/latch.h +./include/postgresql/server/workload/qnode.h +./include/postgresql/server/streaming/init.h +./include/postgresql/server/streaming/launcher.h +./include/postgresql/server/pgxc/barrier.h +./include/postgresql/server/libcomm/libcomm.h +./include/postgresql/server/hotpatch/hotpatch.h +./include/postgresql/server/hotpatch/hotpatch_backend.h +./include/postgresql/server/postmaster/bgwriter.h +./include/postgresql/server/postmaster/pagewriter.h +./include/postgresql/server/replication/heartbeat.h +./include/postgresql/server/access/multi_redo_settings.h +./include/postgresql/server/access/redo_statistic_msg.h +./include/postgresql/server/replication/rto_statistic.h +./include/postgresql/server/replication/walprotocol.h +./include/postgresql/server/storage/mot/jit_def.h +./include/postgresql/server/threadpool/threadpool.h +./include/postgresql/server/threadpool/threadpool_controler.h +./include/postgresql/server/threadpool/threadpool_group.h +./include/postgresql/server/knl/knl_variable.h +./include/postgresql/server/threadpool/threadpool_listener.h +./include/postgresql/server/threadpool/threadpool_sessctl.h +./include/postgresql/server/storage/procsignal.h +./include/postgresql/server/threadpool/threadpool_worker.h +./include/postgresql/server/threadpool/threadpool_scheduler.h +./include/postgresql/server/threadpool/threadpool_stream.h +./include/postgresql/server/replication/dataqueuedefs.h +./include/postgresql/server/gtm/gtm_c.h +./include/postgresql/server/cm/etcdapi.h +./include/postgresql/server/alarm/alarm.h +./include/postgresql/server/access/xact.h +./include/postgresql/server/access/cstore_am.h +./include/postgresql/server/access/cstore_roughcheck_func.h +./include/postgresql/server/access/cstoreskey.h +./include/postgresql/server/storage/cu.h +./include/postgresql/server/vecexecutor/vectorbatch.h +./include/postgresql/server/cstore.h +./include/postgresql/server/storage/cstore/cstore_mem_alloc.h +./include/postgresql/server/access/cstore_minmax_func.h +./include/postgresql/server/storage/custorage.h +./include/postgresql/server/storage/fd.h +./include/postgresql/server/postmaster/aiocompleter.h +./include/postgresql/server/storage/buf/bufmgr.h +./include/postgresql/server/storage/buf/buf_internals.h +./include/postgresql/server/storage/smgr.h +./include/postgresql/server/catalog/pg_am.h +./include/postgresql/server/catalog/pg_class.h +./include/postgresql/server/catalog/pg_index.h +./include/postgresql/server/rewrite/prs2lock.h +./include/postgresql/server/tcop/stmt_retry.h +./include/postgresql/server/catalog/pg_hashbucket_fn.h +./include/postgresql/server/utils/rel_gs.h +./include/postgresql/server/catalog/pg_partition.h +./include/postgresql/server/catalog/pg_hashbucket.h +./include/postgresql/server/catalog/catalog.h +./include/postgresql/server/catalog/catversion.h +./include/postgresql/server/catalog/pg_namespace.h +./include/postgresql/server/utils/partitionmap_gs.h +./include/postgresql/server/access/heapam.h +./include/postgresql/server/storage/pagecompress.h +./include/postgresql/server/replication/bcm.h +./include/postgresql/server/storage/cstore/cstorealloc.h +./include/postgresql/server/storage/cucache_mgr.h +./include/postgresql/server/storage/cache_mgr.h +./include/postgresql/server/nodes/plannodes.h +./include/postgresql/server/foreign/foreign.h +./include/postgresql/server/access/obs/obs_am.h +./include/postgresql/server/storage/buf/buffile.h +./include/postgresql/server/replication/slot.h +./include/postgresql/server/access/obs/eSDKOBS.h +./include/postgresql/server/commands/defrem.h +./include/postgresql/server/optimizer/pruning.h +./include/postgresql/server/nodes/relation.h +./include/postgresql/server/optimizer/bucketinfo.h +./include/postgresql/server/pgxc/nodemgr.h +./include/postgresql/server/bulkload/dist_fdw.h +./include/postgresql/server/bulkload/importerror.h +./include/postgresql/server/commands/gds_stream.h +./include/postgresql/server/bulkload/utils.h +./include/postgresql/server/cjson/cJSON.h +./include/postgresql/server/ssl/gs_openssl_client.h +./include/postgresql/server/funcapi.h +./include/postgresql/server/executor/executor.h +./include/postgresql/server/executor/execdesc.h +./include/postgresql/server/nodes/execnodes.h +./include/postgresql/server/access/genam.h +./include/postgresql/server/nodes/tidbitmap.h +./include/postgresql/server/access/relscan.h +./include/postgresql/server/access/itup.h +./include/postgresql/server/executor/instrument.h +./include/postgresql/server/miscadmin.h +./include/postgresql/server/libpq/libpq-be.h +./include/postgresql/server/libpq/hba.h +./include/postgresql/server/libpq/sha2.h +./include/postgresql/server/utils/anls_opt.h +./include/postgresql/server/pgxc/pgxc.h +./include/postgresql/server/catalog/namespace.h +./include/postgresql/server/commands/trigger.h +./include/postgresql/server/executor/spi.h +./include/postgresql/server/db4ai/db4ai.h +./include/postgresql/server/instruments/gs_stack.h +./include/postgresql/server/instruments/instr_mfchain.h +./include/postgresql/server/access/ustore/undo/knl_uundotype.h +./include/postgresql/server/access/ustore/undo/knl_uundoapi.h +./include/postgresql/server/access/ustore/knl_uheap.h +./include/postgresql/server/access/ustore/knl_utuple.h +./include/postgresql/server/access/ustore/knl_utype.h +./include/postgresql/server/access/ustore/knl_upage.h +./include/postgresql/server/access/ustore/knl_uredo.h +./include/postgresql/server/access/ustore/knl_uundovec.h +./include/postgresql/server/access/ustore/knl_uundorecord.h +./include/postgresql/server/access/ustore/undo/knl_uundoxlog.h +./include/postgresql/server/access/ustore/undo/knl_uundotxn.h +./include/postgresql/server/access/ustore/undo/knl_uundozone.h +./include/postgresql/server/access/ustore/undo/knl_uundospace.h +./include/postgresql/server/communication/commproxy_basic.h +./include/postgresql/server/access/parallel_recovery/page_redo.h +./include/postgresql/server/access/parallel_recovery/spsc_blocking_queue.h +./include/postgresql/server/executor/exec/execdesc.h +./include/postgresql/server/db4ai/matrix.h +./include/postgresql/server/db4ai/scores.h +./include/postgresql/server/access/multixact.h +./include/postgresql/server/access/xlogutils.h +./include/postgresql/server/executor/functions.h +./include/postgresql/server/storage/lock/waitpolicy.h +./include/postgresql/server/storage/page_compression.h +./include/postgresql/server/storage/smgr/knl_usync.h +./include/postgresql/server/storage/smgr/fd.h +./include/postgresql/server/storage/smgr/smgr.h +./include/postgresql/server/storage/sharedfileset.h +./include/postgresql/server/utils/knl_globalsysdbcache.h +./include/postgresql/server/utils/knl_globaldbstatmanager.h +./include/postgresql/server/utils/knl_globalsyscache_common.h +./include/postgresql/server/utils/knl_globalbucketlist.h +./include/postgresql/server/utils/knl_globalsystabcache.h +./include/postgresql/server/utils/knl_globalsystupcache.h +./include/postgresql/server/utils/knl_globaltabdefcache.h +./include/postgresql/server/utils/knl_globalbasedefcache.h +./include/postgresql/server/utils/knl_globalpartdefcache.h +./include/postgresql/server/utils/knl_globalrelmapcache.h +./include/postgresql/server/utils/knl_localsysdbcache.h +./include/postgresql/server/utils/knl_localsystabcache.h +./include/postgresql/server/utils/knl_localsystupcache.h +./include/postgresql/server/utils/knl_localsyscache_common.h +./include/postgresql/server/utils/knl_localtabdefcache.h +./include/postgresql/server/utils/knl_localbasedefcache.h +./include/postgresql/server/utils/knl_localbucketlist.h +./include/postgresql/server/utils/knl_localpartdefcache.h +./include/postgresql/server/utils/pl_global_package_runtime_cache.h +./include/postgresql/server/catalog/pg_tablespace.h +./include/postgresql/server/catalog/pg_subscription.h +./include/postgresql/server/catalog/pg_replication_origin.h +./include/postgresql/server/postmaster/barrier_creator.h +./include/postgresql/server/postmaster/pagerepair.h +./include/postgresql/server/replication/libpqwalreceiver.h +./include/postgresql/server/replication/worker_internal.h +./include/postgresql/server/replication/origin.h +./include/postgresql/server/parser/scanner.h +./include/postgresql/server/parser/keywords.h +./jre/ASSEMBLY_EXCEPTION +./jre/bin/java +./jre/bin/jjs +./jre/bin/keytool +./jre/bin/orbd +./jre/bin/pack200 +./jre/bin/policytool +./jre/bin/rmid +./jre/bin/rmiregistry +./jre/bin/servertool +./jre/bin/tnameserv +./jre/bin/unpack200 +./jre/lib/aarch64/jli/libjli.so +./jre/lib/aarch64/jvm.cfg +./jre/lib/aarch64/libattach.so +./jre/lib/aarch64/libavplugin-ffmpeg-58.so +./jre/lib/aarch64/libawt_headless.so +./jre/lib/aarch64/libawt.so +./jre/lib/aarch64/libawt_xawt.so +./jre/lib/aarch64/libdecora_sse.so +./jre/lib/aarch64/libdt_socket.so +./jre/lib/aarch64/libfontmanager.so +./jre/lib/aarch64/libfxplugins.so +./jre/lib/aarch64/libglassgtk2.so +./jre/lib/aarch64/libglassgtk3.so +./jre/lib/aarch64/libglass.so +./jre/lib/aarch64/libgstreamer-lite.so +./jre/lib/aarch64/libhprof.so +./jre/lib/aarch64/libinstrument.so +./jre/lib/aarch64/libj2gss.so +./jre/lib/aarch64/libj2pcsc.so +./jre/lib/aarch64/libj2pkcs11.so +./jre/lib/aarch64/libjaas_unix.so +./jre/lib/aarch64/libjava_crw_demo.so +./jre/lib/aarch64/libjavafx_font_freetype.so +./jre/lib/aarch64/libjavafx_font_pango.so +./jre/lib/aarch64/libjavafx_font.so +./jre/lib/aarch64/libjavafx_iio.so +./jre/lib/aarch64/libjava.so +./jre/lib/aarch64/libjawt.so +./jre/lib/aarch64/libjdwp.so +./jre/lib/aarch64/libjfxmedia.so +./jre/lib/aarch64/libjfxwebkit.so +./jre/lib/aarch64/libjpeg.so +./jre/lib/aarch64/libjsdt.so +./jre/lib/aarch64/libjsig.so +./jre/lib/aarch64/libjsoundalsa.so +./jre/lib/aarch64/libjsound.so +./jre/lib/aarch64/liblcms.so +./jre/lib/aarch64/libmanagement.so +./jre/lib/aarch64/libmlib_image.so +./jre/lib/aarch64/libnet.so +./jre/lib/aarch64/libnio.so +./jre/lib/aarch64/libnpt.so +./jre/lib/aarch64/libprism_common.so +./jre/lib/aarch64/libprism_es2.so +./jre/lib/aarch64/libprism_sw.so +./jre/lib/aarch64/libsaproc.so +./jre/lib/aarch64/libsctp.so +./jre/lib/aarch64/libsplashscreen.so +./jre/lib/aarch64/libsunec.so +./jre/lib/aarch64/libunpack.so +./jre/lib/aarch64/libverify.so +./jre/lib/aarch64/libzip.so +./jre/lib/aarch64/server/libjvm.so +./jre/lib/aarch64/server/Xusage.txt +./jre/lib/calendars.properties +./jre/lib/charsets.jar +./jre/lib/classlist +./jre/lib/cmm/CIEXYZ.pf +./jre/lib/cmm/GRAY.pf +./jre/lib/cmm/LINEAR_RGB.pf +./jre/lib/cmm/PYCC.pf +./jre/lib/cmm/sRGB.pf +./jre/lib/content-types.properties +./jre/lib/currency.data +./jre/lib/ext/cldrdata.jar +./jre/lib/ext/dnsns.jar +./jre/lib/ext/jaccess.jar +./jre/lib/ext/jfxrt.jar +./jre/lib/ext/localedata.jar +./jre/lib/ext/meta-index +./jre/lib/ext/nashorn.jar +./jre/lib/ext/sunec.jar +./jre/lib/ext/sunjce_provider.jar +./jre/lib/ext/sunpkcs11.jar +./jre/lib/ext/zipfs.jar +./jre/lib/flavormap.properties +./jre/lib/fontconfig.Euler.properties +./jre/lib/fontconfig.properties +./jre/lib/fontconfig.Ubuntu.properties +./jre/lib/fonts/Roboto-Regular.ttf +./jre/lib/hijrah-config-umalqura.properties +./jre/lib/images/cursors/cursors.properties +./jre/lib/images/cursors/invalid32x32.gif +./jre/lib/images/cursors/motif_CopyDrop32x32.gif +./jre/lib/images/cursors/motif_CopyNoDrop32x32.gif +./jre/lib/images/cursors/motif_LinkDrop32x32.gif +./jre/lib/images/cursors/motif_LinkNoDrop32x32.gif +./jre/lib/images/cursors/motif_MoveDrop32x32.gif +./jre/lib/images/cursors/motif_MoveNoDrop32x32.gif +./jre/lib/javafx-mx.jar +./jre/lib/javafx.properties +./jre/lib/jce.jar +./jre/lib/jexec +./jre/lib/jfr/default.jfc +./jre/lib/jfr.jar +./jre/lib/jfr/profile.jfc +./jre/lib/jfxswt.jar +./jre/lib/jsse.jar +./jre/lib/jvm.hprof.txt +./jre/lib/logging.properties +./jre/lib/management-agent.jar +./jre/lib/management/jmxremote.access +./jre/lib/management/jmxremote.password.template +./jre/lib/management/management.properties +./jre/lib/management/snmp.acl.template +./jre/lib/meta-index +./jre/lib/net.properties +./jre/lib/psfontj2d.properties +./jre/lib/psfont.properties.ja +./jre/lib/resources.jar +./jre/lib/rt.jar +./jre/lib/security/blacklisted.certs +./jre/lib/security/cacerts +./jre/lib/security/java.policy +./jre/lib/security/java.security +./jre/lib/security/policy/limited/local_policy.jar +./jre/lib/security/policy/limited/US_export_policy.jar +./jre/lib/security/policy/unlimited/local_policy.jar +./jre/lib/security/policy/unlimited/US_export_policy.jar +./jre/lib/sound.properties +./jre/lib/tzdb.dat +./jre/LICENSE +./jre/THIRD_PARTY_README +[client] +./bin/gsql +./bin/gs_dump +./bin/gs_dumpall +./bin/gs_restore +./bin/gs_basebackup +./bin/gs_probackup +./lib/postgresql/latin2_and_win1250.so +./lib/postgresql/euc2004_sjis2004.so +./lib/postgresql/euc_kr_and_mic.so +./lib/postgresql/utf8_and_uhc.so +./lib/postgresql/euc_tw_and_big5.so +./lib/postgresql/cyrillic_and_mic.so +./lib/postgresql/utf8_and_johab.so +./lib/postgresql/utf8_and_gb18030.so +./lib/postgresql/pgxs/src/makefiles/pgxs.mk +./lib/postgresql/pgxs/src/Makefile.shlib +./lib/postgresql/pgxs/src/Makefile.port +./lib/postgresql/pgxs/src/nls-global.mk +./lib/postgresql/pgxs/src/Makefile.global +./lib/postgresql/pgxs/config/install-sh +./lib/postgresql/euc_cn_and_mic.so +./lib/postgresql/latin_and_mic.so +./lib/postgresql/utf8_and_sjis2004.so +./lib/postgresql/utf8_and_euc_jp.so +./lib/postgresql/utf8_and_sjis.so +./lib/postgresql/utf8_and_cyrillic.so +./lib/postgresql/utf8_and_euc_kr.so +./lib/postgresql/ascii_and_mic.so +./lib/postgresql/utf8_and_iso8859_1.so +./lib/postgresql/euc_jp_and_sjis.so +./lib/postgresql/dict_snowball.so +./lib/postgresql/utf8_and_ascii.so +./lib/postgresql/utf8_and_euc_tw.so +./lib/postgresql/utf8_and_iso8859.so +./lib/postgresql/utf8_and_win.so +./lib/postgresql/utf8_and_euc_cn.so +./lib/postgresql/utf8_and_gbk.so +./lib/postgresql/utf8_and_euc2004.so +./lib/postgresql/utf8_and_big5.so +./lib/postgresql/java/pljava.jar +./lib/libpljava.so +./lib/libpq.a +./lib/libpq.so +./lib/libpq.so.5 +./lib/libpq.so.5.5 +./lib/libpq_ce.so +./lib/libpq_ce.so.5 +./lib/libpq_ce.so.5.5 +./lib/libgauss_cl_jni.so +./lib/libconfig.so* +./lib/libcrypto.so* +./lib/libstdc++.so.6 +./lib/libssl.so* +./lib/libpgport_tool.so +./lib/libpgport_tool.so.1 +./lib/libgssapi_krb5_gauss.so* +./lib/libgssrpc_gauss.so* +./lib/libk5crypto_gauss.so* +./lib/libkrb5support_gauss.so* +./lib/libkrb5_gauss.so* +./lib/libcom_err_gauss.so* +./lib/libcjson.so* +[libpq] +./lib/libpq.a +./lib/libpq.so +./lib/libpq.so.5 +./lib/libpq.so.5.5 +./lib/libpq_ce.so +./lib/libpq_ce.so.5 +./lib/libpq_ce.so.5.5 +./lib/libgauss_cl_jni.so +./lib/libconfig.so* +./lib/libcrypto.so* +./lib/libstdc++.so.6 +./lib/libssl.so* +./lib/libpgport_tool.so +./lib/libpgport_tool.so.1 +./lib/libgssapi_krb5_gauss.so* +./lib/libgssrpc_gauss.so* +./lib/libk5crypto_gauss.so* +./lib/libkrb5support_gauss.so* +./lib/libkrb5_gauss.so* +./lib/libcom_err_gauss.so* +./lib/libcjson.so* +./include/gs_thread.h +./include/gs_threadlocal.h +./include/postgres_ext.h +./include/libpq-fe.h +./include/libpq-events.h +./include/libpq/libpq-fs.h +[header] +./include/libpq-fe.h +./include/postgres_ext.h +./include/gs_thread.h +./include/gs_threadlocal.h +./include/pg_config.h +./include/pg_config_manual.h +./include/pg_config_os.h +./include/cm_config.h +./include/c.h +./include/port.h +./include/libpq-int.h +./include/pqcomm.h +./include/pqexpbuffer.h +./include/xlogdefs.h +./include/cm-libpq-fe.h diff --git a/build/script/utils/common.sh b/build/script/utils/common.sh index d26786c80..f85d99d6e 100644 --- a/build/script/utils/common.sh +++ b/build/script/utils/common.sh @@ -104,6 +104,10 @@ else gcc_version="7.3" fi +if [ "$PLATFORM_ARCH"X == "loongarch64"X ];then + gcc_version="8.3" +fi + if [ "$PLATFORM_ARCH"X == "aarch64"X ] && [ "$gcc_version" == "10.3" ]; then gcc_sub_version="1" else diff --git a/build/script/utils/make_compile.sh b/build/script/utils/make_compile.sh index d9e9650f2..82a16282d 100644 --- a/build/script/utils/make_compile.sh +++ b/build/script/utils/make_compile.sh @@ -144,7 +144,7 @@ function install_gaussdb() echo "Begin configure." >> "$LOG_FILE" 2>&1 chmod 755 configure - if [ "$product_mode"x == "opengauss"x ]; then + if [ "$product_mode"x == "opengauss"x -a "$PLATFORM_ARCH"x != "loongarch64"x ]; then enable_readline="--with-readline" else enable_readline="--without-readline" @@ -158,6 +158,17 @@ function install_gaussdb() shared_opt="--gcc-version=${gcc_version}.${gcc_sub_version} --prefix="${BUILD_DIR}" --3rd=${binarylib_dir} --enable-thread-safety ${enable_readline} ${with_tassl} --without-zlib" if [ "$product_mode"x == "opengauss"x ]; then + GAUSSDB_EXTRA_FLAGS=" " + + if [[ "$PLATFORM_ARCH"x == "x86_64"x || "$PLATFORM_ARCH"x == "aarch64"x ]] ; then + extra_config_opt+=" --enable-mot " + fi + + if [ "$PLATFORM_ARCH"x = "loongarch64"x ] ; then + GAUSSDB_EXTRA_FLAGS+=" -D__USE_SPINLOCK" + extra_config_opt+=" --enable-llvm=no --disable-jemalloc " + fi + if [ "$version_mode"x == "release"x ]; then # configure -D__USE_NUMA -D__ARM_LSE with arm opengauss mode if [ "$PLATFORM_ARCH"X == "aarch64"X ] ; then @@ -165,15 +176,15 @@ function install_gaussdb() GAUSSDB_EXTRA_FLAGS=" -D__USE_NUMA -D__ARM_LSE" fi - ./configure $shared_opt CFLAGS="-O2 -g3 ${GAUSSDB_EXTRA_FLAGS}" --enable-mot CC=g++ $extra_config_opt >> "$LOG_FILE" 2>&1 + ./configure $shared_opt CFLAGS="-O2 -g3 ${GAUSSDB_EXTRA_FLAGS}" CC=g++ $extra_config_opt >> "$LOG_FILE" 2>&1 elif [ "$version_mode"x == "memcheck"x ]; then - ./configure $shared_opt CFLAGS="-O0" --enable-mot --enable-debug --enable-cassert --enable-memory-check CC=g++ $extra_config_opt >> "$LOG_FILE" 2>&1 + ./configure $shared_opt CFLAGS="-O0" --enable-debug --enable-cassert --enable-memory-check CC=g++ $extra_config_opt >> "$LOG_FILE" 2>&1 elif [ "$version_mode"x == "fiurelease"x ]; then - ./configure $shared_opt CFLAGS="-O2 -g3 ${GAUSSDB_EXTRA_FLAGS}" --enable-mot --disable-jemalloc CC=g++ $extra_config_opt >> "$LOG_FILE" 2>&1 + ./configure $shared_opt CFLAGS="-O2 -g3 ${GAUSSDB_EXTRA_FLAGS}" --disable-jemalloc CC=g++ $extra_config_opt >> "$LOG_FILE" 2>&1 elif [ "$version_mode"x == "fiudebug"x ]; then - ./configure $shared_opt CFLAGS="-O0 ${GAUSSDB_EXTRA_FLAGS}" --enable-mot --enable-debug --enable-cassert --disable-jemalloc CC=g++ $extra_config_opt >> "$LOG_FILE" 2>&1 + ./configure $shared_opt CFLAGS="-O0 ${GAUSSDB_EXTRA_FLAGS}" --enable-debug --enable-cassert --disable-jemalloc CC=g++ $extra_config_opt >> "$LOG_FILE" 2>&1 else - ./configure $shared_opt CFLAGS="-O0 ${GAUSSDB_EXTRA_FLAGS}" --enable-mot --enable-debug --enable-cassert CC=g++ $extra_config_opt >> "$LOG_FILE" 2>&1 + ./configure $shared_opt CFLAGS="-O0 ${GAUSSDB_EXTRA_FLAGS}" --enable-debug --enable-cassert CC=g++ $extra_config_opt >> "$LOG_FILE" 2>&1 fi elif [ "$product_mode"x == "lite"x ]; then shared_opt="--gcc-version=${gcc_version}.${gcc_sub_version} --prefix="${BUILD_DIR}" --3rd=${binarylib_dir} --enable-thread-safety ${enable_readline} ${with_tassl} --without-zlib --without-gssapi --without-krb5" diff --git a/cmake/src/build_options.cmake b/cmake/src/build_options.cmake index b357cd106..819e53bd3 100755 --- a/cmake/src/build_options.cmake +++ b/cmake/src/build_options.cmake @@ -62,6 +62,8 @@ option(ENABLE_NUMA "enable numa,the old is --enable-numa" ON) option(ENABLE_LSE "enable lse,the old is --enable-lse" ON) option(ENABLE_MYSQL_FDW "enable export or import data with mysql,the old is --enable-mysql-fdw" OFF) option(ENABLE_ORACLE_FDW "enable export or import data with oracle,the old is --enable-oracle-fdw" OFF) +option(ENABLE_BBOX "enable bbox,the old is --enable-bbox " ON) +option(ENABLE_JEMALLOC "enable jemalloc,the old is --enable-jemalloc " ON) option(BUILD_BY_CMAKE "the BUILD_BY_CMAKE is new,used in distribute pg_regress.cpp" ON) option(DEBUG_UHEAP "collect USTORE statistics" OFF) option(MAX_ALLOC_SEGNUM "max alloc xlog seg num in extreme_rto" 4) diff --git a/config/config.guess b/config/config.guess index d622a44e5..15f22ac76 100644 --- a/config/config.guess +++ b/config/config.guess @@ -138,6 +138,33 @@ UNAME_RELEASE=`(uname -r) 2>/dev/null` || UNAME_RELEASE=unknown UNAME_SYSTEM=`(uname -s) 2>/dev/null` || UNAME_SYSTEM=unknown UNAME_VERSION=`(uname -v) 2>/dev/null` || UNAME_VERSION=unknown +case "$UNAME_SYSTEM" in +Linux|GNU|GNU/*) + # If the system lacks a compiler, then just pick glibc. + # We could probably try harder. + LIBC=gnu + + eval "$set_cc_for_build" + cat <<-EOF > "$dummy.c" + #include + #if defined(__UCLIBC__) + LIBC=uclibc + #elif defined(__dietlibc__) + LIBC=dietlibc + #else + LIBC=gnu + #endif + EOF + eval "`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^LIBC' | sed 's, ,,g'`" + + # If ldd exists, use it to detect musl libc. + if command -v ldd >/dev/null && \ + ldd --version 2>&1 | grep -q ^musl + then + LIBC=musl + fi + ;; +esac # Note: order is significant - the case branches are not exclusive. case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in @@ -933,6 +960,9 @@ EOF m68*:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-gnu exit ;; + loongarch*:Linux:*:*) + echo "$UNAME_MACHINE"-linux-"$LIBC" + exit ;; mips:Linux:*:* | mips64:Linux:*:*) eval $set_cc_for_build sed 's/^ //' << EOF >$dummy.c diff --git a/config/config.sub b/config/config.sub index 02df9f8bf..6e5c23c53 100644 --- a/config/config.sub +++ b/config/config.sub @@ -274,6 +274,7 @@ case $basic_machine in | lm32 \ | m32c | m32r | m32rle | m68000 | m68k | m88k \ | maxq | mb | microblaze | mcore | mep | metag \ + | loongarch32 | loongarch64 \ | mips | mipsbe | mipseb | mipsel | mipsle \ | mips16 \ | mips64 | mips64el \ @@ -390,6 +391,7 @@ case $basic_machine in | m32c-* | m32r-* | m32rle-* \ | m68000-* | m680[012346]0-* | m68360-* | m683?2-* | m68k-* \ | m88110-* | m88k-* | maxq-* | mcore-* | metag-* | microblaze-* \ + | loongarch32-* | loongarch64-* \ | mips-* | mipsbe-* | mipseb-* | mipsel-* | mipsle-* \ | mips16-* \ | mips64-* | mips64el-* \ diff --git a/configure b/configure index da13baaca..ccba40b73 100755 --- a/configure +++ b/configure @@ -755,6 +755,7 @@ enable_privategauss enable_multiple_nodes enable_lite_mode enable_mot +enable_bbox enable_memory_check enable_mysql_fdw enable_oracle_fdw @@ -841,6 +842,7 @@ enable_privategauss enable_multiple_nodes enable_lite_mode enable_mot +enable_bbox enable_memory_check enable_mysql_fdw enable_oracle_fdw @@ -2462,7 +2464,10 @@ fi # Add non-standard directories to the library search path # - +PLATFORM_ARCH=$(uname -p) +if [ "$PLATFORM_ARCH"X == "unknown"X ]; then + PLATFORM_ARCH=$(uname -m) +fi # Check whether --with-libraries was given. if test "${with_libraries+set}" = set; then @@ -3237,6 +3242,34 @@ _ACEOF fi +# Check whether --enable-bbox was given. +if test "${enable_bbox+set}" = set; then + enableval=$enable_bbox; + case $enableval in + yes) + : + ;; + no) + : + ;; + *) + enable_bbox=no + ;; + esac + +else + enable_bbox=no + +fi + +if test "$enable_bbox" = yes; then + +cat >>confdefs.h <<\_ACEOF +#define ENABLE_BBOX 1 +_ACEOF + +fi + # # --with-openeuler-os enable # @@ -5885,14 +5918,13 @@ if test "${enable_llvm+set}" = set; then if test "$enable_lite_mode" = yes; then { $as_echo "$as_me:$LINENO: enable_lite_mode is open, llvm will close" >&5 $as_echo "$as_me: enable_lite_mode is open, llvm will close" >&2;} +enable_llvm=no else -cat >>confdefs.h <<\_ACEOF -#define ENABLE_LLVM_COMPILE 1 -_ACEOF +enable_llvm=yes fi ;; no) - : +enable_llvm=no ;; *) { { $as_echo "$as_me:$LINENO: error: no argument expected for --enable-llvm option" >&5 @@ -5905,14 +5937,18 @@ else if test "$enable_lite_mode" = yes; then { $as_echo "$as_me:$LINENO: enable_lite_mode is open, llvm will close" >&5 $as_echo "$as_me: enable_lite_mode is open, llvm will close" >&2;} +enable_llvm=no else -cat >>confdefs.h <<\_ACEOF -#define ENABLE_LLVM_COMPILE 1 -_ACEOF +enable_llvm=yes fi fi +if test "$enable_llvm" = yes; then +cat >>confdefs.h <<\_ACEOF +#define ENABLE_LLVM_COMPILE 1 +_ACEOF + llvm_version_str='10.0.0' if [[ ! -z "${with_3rdpartydir}" ]] && [[ "$enable_lite_mode" != yes ]]; then llvm_version_str=`${with_3rdpartydir}/kernel/dependency/llvm/comm/bin/llvm-config --version` @@ -5924,6 +5960,7 @@ cat >>confdefs.h <<_ACEOF #define LLVM_MAJOR_VERSION $llvm_major_version #define LLVM_MINOR_VERSION $llvm_minor_version _ACEOF +fi # # Enable ut diff --git a/src/Makefile.global.in b/src/Makefile.global.in index 4e04f5b01..2fc6ea503 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -175,6 +175,8 @@ enable_privategauss = @enable_privategauss@ enable_multiple_nodes = @enable_multiple_nodes@ enable_lite_mode = @enable_lite_mode@ enable_mot = @enable_mot@ +enable_bbox = @enable_bbox@ +enable_llvm = @enable_llvm@ enable_mysql_fdw = @enable_mysql_fdw@ enable_oracle_fdw = @enable_oracle_fdw@ enable_memory_check = @enable_memory_check@ @@ -496,12 +498,19 @@ LIBHOTPATCH_TOOL_PATH = $(LIBHOTPATCH_HOME)/tool ############################################################################# # llvm component ############################################################################# -LIBLLVM_BIN = $(LIBLLVM_HOME)/bin -LIBLLVM_INCLUDE_PATH = $(LIBLLVM_HOME)/include -LIBLLVM_LIB_PATH = $(LIBLLVM_HOME)/lib -LLVM_CONFIG = $(LIBLLVM_BIN)/llvm-config -ifneq ($(enable_lite_mode), yes) - LLVM_LIBS = $(shell $(LLVM_CONFIG) --libs) + +ifeq ($(enable_llvm), yes) + LIBLLVM_BIN = $(LIBLLVM_HOME)/bin + LIBLLVM_INCLUDE_PATH = $(LIBLLVM_HOME)/include + LIBLLVM_LIB_PATH = $(LIBLLVM_HOME)/lib + LLVM_CONFIG = $(LIBLLVM_BIN)/llvm-config + ifneq ($(enable_lite_mode), yes) + LLVM_LIBS = $(shell $(LLVM_CONFIG) --libs) + endif +else + LIBLLVM_INCLUDE_PATH = /usr/include + LIBLLVM_LIB_PATH = /usr/lib + LLVM_LIBS = endif ############################################################################# @@ -869,7 +878,9 @@ ifeq ($(SUPPORT_HOTPATCH), yes) LDFLAGS += -L$(LIBHOTPATCH_LIB_PATH) endif +ifeq ($(enable_bbox), yes) LDFLAGS += -L$(BBOX_LIB_PATH) +endif LDFLAGS += -L$(LIBODBC_LIB_PATH) LDFLAGS += -L$(LIBOBS_LIB_PATH) LDFLAGS += -L$(KERBEROS_LIB_PATH) diff --git a/src/common/backend/utils/misc/guc.cpp b/src/common/backend/utils/misc/guc.cpp index add4504fd..ac8966769 100755 --- a/src/common/backend/utils/misc/guc.cpp +++ b/src/common/backend/utils/misc/guc.cpp @@ -42,7 +42,9 @@ #ifdef ENABLE_WHITEBOX #include "access/ustore/knl_whitebox_test.h" #endif +#ifdef ENABLE_BBOX #include "gs_bbox.h" +#endif #include "catalog/namespace.h" #include "catalog/pgxc_group.h" #include "catalog/storage_gtt.h" @@ -1678,7 +1680,11 @@ static void InitConfigureNamesBool() &u_sess->attr.attr_common.enable_bbox_dump, true, NULL, +#ifdef ENABLE_BBOX assign_bbox_coredump, +#else + NULL, +#endif /* ENABLE_BBOX */ NULL}, {{"enable_default_index_deduplication", PGC_POSTMASTER, @@ -3718,9 +3724,15 @@ static void InitConfigureNamesString() NULL}, &u_sess->attr.attr_common.bbox_dump_path, "", +#ifdef ENABLE_BBOX check_bbox_corepath, assign_bbox_corepath, show_bbox_dump_path}, +#else + NULL, + NULL, + NULL}, +#endif /* ENABLE_BBOX */ {{"alarm_component", PGC_POSTMASTER, @@ -3984,9 +3996,15 @@ static void InitConfigureNamesString() GUC_LIST_INPUT}, &g_instance.attr.attr_common.bbox_blacklist_items, "", +#ifdef ENABLE_BBOX check_bbox_blacklist, assign_bbox_blacklist, show_bbox_blacklist}, +#else + NULL, + NULL, + NULL}, +#endif /* ENABLE_BBOX */ {{"track_stmt_stat_level", PGC_USERSET, diff --git a/src/common/backend/utils/misc/guc/guc_memory.cpp b/src/common/backend/utils/misc/guc/guc_memory.cpp index d9d6a0cc4..716f2b1b6 100644 --- a/src/common/backend/utils/misc/guc/guc_memory.cpp +++ b/src/common/backend/utils/misc/guc/guc_memory.cpp @@ -35,7 +35,9 @@ #include "access/twophase.h" #include "access/xact.h" #include "access/xlog.h" +#ifdef ENABLE_BBOX #include "gs_bbox.h" +#endif #include "catalog/namespace.h" #include "catalog/pgxc_group.h" #include "catalog/storage_gtt.h" diff --git a/src/common/backend/utils/misc/guc/guc_network.cpp b/src/common/backend/utils/misc/guc/guc_network.cpp index 14104f334..66915f237 100755 --- a/src/common/backend/utils/misc/guc/guc_network.cpp +++ b/src/common/backend/utils/misc/guc/guc_network.cpp @@ -36,7 +36,9 @@ #include "access/twophase.h" #include "access/xact.h" #include "access/xlog.h" +#ifdef ENABLE_BBOX #include "gs_bbox.h" +#endif #include "catalog/namespace.h" #include "catalog/pgxc_group.h" #include "catalog/storage_gtt.h" diff --git a/src/common/backend/utils/misc/guc/guc_resource.cpp b/src/common/backend/utils/misc/guc/guc_resource.cpp index a417b942e..4bbc6bf5d 100644 --- a/src/common/backend/utils/misc/guc/guc_resource.cpp +++ b/src/common/backend/utils/misc/guc/guc_resource.cpp @@ -35,7 +35,9 @@ #include "access/twophase.h" #include "access/xact.h" #include "access/xlog.h" +#ifdef ENABLE_BBOX #include "gs_bbox.h" +#endif #include "catalog/namespace.h" #include "catalog/pgxc_group.h" #include "catalog/storage_gtt.h" diff --git a/src/common/backend/utils/misc/guc/guc_security.cpp b/src/common/backend/utils/misc/guc/guc_security.cpp index 9e6a97a37..7e559a8fb 100755 --- a/src/common/backend/utils/misc/guc/guc_security.cpp +++ b/src/common/backend/utils/misc/guc/guc_security.cpp @@ -35,7 +35,9 @@ #include "access/twophase.h" #include "access/xact.h" #include "access/xlog.h" +#ifdef ENABLE_BBOX #include "gs_bbox.h" +#endif #include "catalog/namespace.h" #include "catalog/pgxc_group.h" #include "catalog/storage_gtt.h" diff --git a/src/common/backend/utils/misc/guc/guc_sql.cpp b/src/common/backend/utils/misc/guc/guc_sql.cpp index edc4298c7..ecff10d5a 100755 --- a/src/common/backend/utils/misc/guc/guc_sql.cpp +++ b/src/common/backend/utils/misc/guc/guc_sql.cpp @@ -38,7 +38,9 @@ #include "access/xlog.h" #include "access/ustore/knl_whitebox_test.h" #include "access/ubtree.h" +#ifdef ENABLE_BBOX #include "gs_bbox.h" +#endif #include "catalog/namespace.h" #include "catalog/pgxc_group.h" #include "catalog/storage_gtt.h" diff --git a/src/common/backend/utils/misc/guc/guc_storage.cpp b/src/common/backend/utils/misc/guc/guc_storage.cpp index d7201f1a7..2aad73467 100755 --- a/src/common/backend/utils/misc/guc/guc_storage.cpp +++ b/src/common/backend/utils/misc/guc/guc_storage.cpp @@ -37,7 +37,9 @@ #include "access/twophase.h" #include "access/xact.h" #include "access/xlog.h" +#ifdef ENABLE_BBOX #include "gs_bbox.h" +#endif #include "catalog/namespace.h" #include "catalog/pgxc_group.h" #include "catalog/storage_gtt.h" diff --git a/src/common/port/CMakeLists.txt b/src/common/port/CMakeLists.txt index 58e05213d..b9a6a282a 100755 --- a/src/common/port/CMakeLists.txt +++ b/src/common/port/CMakeLists.txt @@ -90,6 +90,10 @@ if("${BUILD_TUPLE}" STREQUAL "aarch64") set(TGT_crc32_arm_parallel_SRC ${CMAKE_CURRENT_SOURCE_DIR}/crc32_arm_parallel.S) endif() +if("${BUILD_TUPLE}" STREQUAL "loongarch64") + list(REMOVE_ITEM TGT_port_SRC ${CMAKE_CURRENT_SOURCE_DIR}/pg_crc32c_choose.cpp ${CMAKE_CURRENT_SOURCE_DIR}/pg_crc32c_sse42.cpp) +endif() + SET(TGT_pgport_INC ${PROJECT_SRC_DIR}/common/backend ${PROJECT_SRC_DIR}/common/port @@ -158,6 +162,11 @@ if("${BUILD_TUPLE}" STREQUAL "aarch64") list(REMOVE_ITEM TGT_pgport_srv_SRC ${CMAKE_CURRENT_SOURCE_DIR}/port_srv/pg_crc32c_sb8.cpp ${CMAKE_CURRENT_SOURCE_DIR}/port_srv/pg_crc32c_sse42.cpp) set(TGT_crc32_arm_parallel_srv_SRC ${CMAKE_CURRENT_SOURCE_DIR}/port_srv/crc32_arm_parallel.S) endif() + +if("${BUILD_TUPLE}" STREQUAL "loongarch64") + list(REMOVE_ITEM TGT_pgport_srv_SRC ${CMAKE_CURRENT_SOURCE_DIR}/port_srv/pg_crc32c_choose.cpp ${CMAKE_CURRENT_SOURCE_DIR}/port_srv/pg_crc32c_sse42.cpp) +endif() + SET(TGT_pgport_srv_INC ${PROJECT_SRC_DIR}/common/backend ${PROJECT_SRC_DIR}/common/port diff --git a/src/common/port/Makefile b/src/common/port/Makefile index 38bdcf7bd..c3d78c261 100644 --- a/src/common/port/Makefile +++ b/src/common/port/Makefile @@ -44,7 +44,7 @@ ifneq "$(MAKECMDGOALS)" "clean" endif endif -OBJS = $(LIBOBJS) pg_crc32c_sse42.o pg_crc32c_sb8.o pg_crc32c_choose.o chklocale.o dirmod.o erand48.o exec.o fls.o inet_net_ntop.o \ +OBJS = $(LIBOBJS) chklocale.o dirmod.o erand48.o exec.o fls.o inet_net_ntop.o \ noblock.o path.o pg_bitutils.o pgcheckdir.o pgmkdirp.o pgsleep.o \ pgstrcasecmp.o qsort.o qsort_arg.o sprompt.o thread.o flock.o pgstrcasestr.o\ gs_thread.o gs_env_r.o gs_getopt_r.o \ @@ -54,15 +54,12 @@ OBJS = $(LIBOBJS) pg_crc32c_sse42.o pg_crc32c_sb8.o pg_crc32c_choose.o chklocale $(top_builddir)/src/gausskernel/storage/file/fio_device.o $(top_builddir)/src/gausskernel/storage/dss/fio_dss.o ifeq "${host_cpu}" "aarch64" -OBJS = $(LIBOBJS) pg_crc32c_choose.o chklocale.o dirmod.o erand48.o exec.o fls.o inet_net_ntop.o \ - noblock.o path.o pg_bitutils.o pgcheckdir.o pgmkdirp.o pgsleep.o \ - pgstrcasecmp.o qsort.o qsort_arg.o sprompt.o thread.o flock.o pgstrcasestr.o\ - gs_thread.o gs_env_r.o gs_getopt_r.o \ - gs_readdir.o gs_strerror.o gs_syscall_lock.o \ - gs_system.o \ - cipher.o tool_common.o \ - $(top_builddir)/src/gausskernel/storage/file/fio_device.o $(top_builddir)/src/gausskernel/storage/dss/fio_dss.o -endif +OBJS += pg_crc32c_choose.o +else ifeq "${host_cpu}" "x86_64" +OBJS += pg_crc32c_sse42.o pg_crc32c_sb8.o pg_crc32c_choose.o +else +OBJS += pg_crc32c_sb8.o +endif # foo_srv.o and foo.o are both built from foo.c, but only foo.o has -DFRONTEND OBJS_SRV = $(OBJS:%.o=%_srv.o) diff --git a/src/gausskernel/CMakeLists.txt b/src/gausskernel/CMakeLists.txt index 21b9e9e44..995960c6d 100755 --- a/src/gausskernel/CMakeLists.txt +++ b/src/gausskernel/CMakeLists.txt @@ -138,6 +138,12 @@ set(gaussdb_objects $ ) +if("${ENABLE_BBOX}" STREQUAL "OFF") +list(REMOVE_ITEM gaussdb_objects + $ +) +endif() + if("${ENABLE_MULTIPLE_NODES}" STREQUAL "OFF") list(APPEND gaussdb_objects $ @@ -292,11 +298,15 @@ if(${USE_LIBXML}) endif() if(NOT "${ENABLE_LITE_MODE}" STREQUAL "ON") - list(APPEND gaussdb_LINK_LIBS -lz -lminiunz -leSDKOBS -leSDKLogAPI -lpcre -liconv -lnghttp2 -llog4cpp -lcurl -llz4 -lcjson -l${JEMALLOC_LIB_NAME} -lcgroup -lzstd -lcom_err_gauss -lgssapi_krb5_gauss -lkrb5_gauss -lgssrpc_gauss -lk5crypto_gauss -lkadm5clnt_mit -lkadm5srv_mit -lkdb5 -lkrb5support_gauss -lstdc++ -lboost_thread -lboost_chrono -lboost_system -lboost_atomic -lxml2 -laio -lncurses -ltinfo) + list(APPEND gaussdb_LINK_LIBS -lz -lminiunz -leSDKOBS -leSDKLogAPI -lpcre -liconv -lnghttp2 -llog4cpp -lcurl -llz4 -lcjson -l${JEMALLOC_LIB_NAME} -lcgroup -lzstd -lcom_err_gauss -lgssapi_krb5_gauss -lkrb5_gauss -lgssrpc_gauss -lk5crypto_gauss -lkadm5clnt_mit -lkadm5srv_mit -lkdb5 -lkrb5support_gauss -lstdc++ -lboost_thread -lboost_chrono -lboost_system -lboost_atomic -lxml2 -laio -lncurses -ltinfo -latomic) else() list(APPEND gaussdb_LINK_LIBS -lz -lminiunz -lcurl -llz4 -lcjson -l${JEMALLOC_LIB_NAME} -lcgroup -lzstd -lncurses -ltinfo -lboost_thread -lboost_chrono -lboost_system -lboost_atomic) endif() +if(NOT "${ENABLE_JEMALLOC}" STREQUAL "ON") + list(REMOVE_ITEM gaussdb_LINK_LIBS -l${JEMALLOC_LIB_NAME}) +endif() + include_directories( ${LIBCURL_INCLUDE_PATH} ${LIBOPENSSL_INCLUDE_PATH} diff --git a/src/gausskernel/Makefile b/src/gausskernel/Makefile index b54aaad78..ddfaf1216 100755 --- a/src/gausskernel/Makefile +++ b/src/gausskernel/Makefile @@ -145,6 +145,7 @@ ifeq ($(SUPPORT_HOTPATCH), yes) CRC_CHECK=$(LIBHOTPATCH_TOOL_PATH)/makepatch -CHECK -CRC gaussdb > $(CRC_LOG) endif +LIBS += -latomic ########################################################################## # append lz4 for compression : lz4.a or lz4.a ############################################################################ @@ -657,10 +658,11 @@ endif ifeq ($(enable_lite_mode), no) cp -r $(with_jdk)/jre/* '$(DESTDIR)$(bindir)/../jre/' - +ifneq ($(host_cpu), loongarch64) cp $(PLJAVA_LIB_PATH)/* '$(DESTDIR)$(libdir)/' cp $(PLJAVA_JAR_PATH)/$(JARPLJAVA) '$(DESTDIR)$(pkglibdir)/java/' cp $(PLJAVA_HOME)/udstools.py '$(DESTDIR)$(datadir)/tmp/' +endif endif cp '$(top_builddir)/src/include/ssl/openssl_gsql.cnf' '$(DESTDIR)$(datadir)/../sslcert/gsql/openssl.cnf' diff --git a/src/gausskernel/Makefile_for_llt b/src/gausskernel/Makefile_for_llt index 7dfb9ea7a..e593bdd7d 100755 --- a/src/gausskernel/Makefile_for_llt +++ b/src/gausskernel/Makefile_for_llt @@ -74,9 +74,11 @@ OBJS = $(SUBDIROBJS) $(LOCALOBJS) \ LIBS := $(filter-out -lpgport, $(LIBS)) $(LDAP_LIBS_BE) # the link options for llvm -LLVM_ZLIB_NAME=z -LLVMLIBS := -L$(LIBLLVM_LIB_PATH) -L$(LIBTINFO_LIB_PATH) -l$(LLVM_ZLIB_NAME) -pthread -lncurses -lrt -ldl -lm -lLLVMInstrumentation -lLLVMIRReader -lLLVMAsmParser -lLLVMDebugInfo -lLLVMOption -lLLVMLTO -lLLVMLinker -lLLVMipo -lLLVMVectorize -lLLVMBitWriter -lLLVMBitReader -lLLVMTableGen -lLLVMR600CodeGen -lLLVMR600Desc -lLLVMR600Info -lLLVMR600AsmPrinter -lLLVMSystemZDisassembler -lLLVMSystemZCodeGen -lLLVMSystemZAsmParser -lLLVMSystemZDesc -lLLVMSystemZInfo -lLLVMSystemZAsmPrinter -lLLVMHexagonCodeGen -lLLVMHexagonAsmPrinter -lLLVMHexagonDesc -lLLVMHexagonInfo -lLLVMNVPTXCodeGen -lLLVMNVPTXDesc -lLLVMNVPTXInfo -lLLVMNVPTXAsmPrinter -lLLVMCppBackendCodeGen -lLLVMCppBackendInfo -lLLVMMSP430CodeGen -lLLVMMSP430Desc -lLLVMMSP430Info -lLLVMMSP430AsmPrinter -lLLVMXCoreDisassembler -lLLVMXCoreCodeGen -lLLVMXCoreDesc -lLLVMXCoreInfo -lLLVMXCoreAsmPrinter -lLLVMMipsDisassembler -lLLVMMipsCodeGen -lLLVMMipsAsmParser -lLLVMMipsDesc -lLLVMMipsInfo -lLLVMMipsAsmPrinter -lLLVMARMDisassembler -lLLVMARMCodeGen -lLLVMARMAsmParser -lLLVMARMDesc -lLLVMARMInfo -lLLVMARMAsmPrinter -lLLVMAArch64Disassembler -lLLVMAArch64CodeGen -lLLVMAArch64AsmParser -lLLVMAArch64Desc -lLLVMAArch64Info -lLLVMAArch64AsmPrinter -lLLVMAArch64Utils -lLLVMPowerPCCodeGen -lLLVMPowerPCAsmParser -lLLVMPowerPCDesc -lLLVMPowerPCInfo -lLLVMPowerPCAsmPrinter -lLLVMSparcCodeGen -lLLVMSparcDesc -lLLVMSparcInfo -lLLVMX86Disassembler -lLLVMX86AsmParser -lLLVMX86CodeGen -lLLVMSelectionDAG -lLLVMAsmPrinter -lLLVMX86Desc -lLLVMX86Info -lLLVMX86AsmPrinter -lLLVMX86Utils -lLLVMMCDisassembler -lLLVMMCParser -lLLVMInterpreter -lLLVMMCJIT -lLLVMJIT -lLLVMCodeGen -lLLVMObjCARCOpts -lLLVMScalarOpts -lLLVMInstCombine -lLLVMTransformUtils -lLLVMipa -lLLVMAnalysis -lLLVMRuntimeDyld -lLLVMExecutionEngine -lLLVMTarget -lLLVMMC -lLLVMObject -lLLVMCore -lLLVMSupport -ltinfo - +ifeq ($(enable_llvm), yes) + # the link options for llvm + LLVM_ZLIB_NAME=z + LLVMLIBS := -L$(LIBLLVM_LIB_PATH) -L$(LIBTINFO_LIB_PATH) -l$(LLVM_ZLIB_NAME) -pthread -lncurses -lrt -ldl -lm -lLLVMInstrumentation -lLLVMIRReader -lLLVMAsmParser -lLLVMDebugInfo -lLLVMOption -lLLVMLTO -lLLVMLinker -lLLVMipo -lLLVMVectorize -lLLVMBitWriter -lLLVMBitReader -lLLVMTableGen -lLLVMR600CodeGen -lLLVMR600Desc -lLLVMR600Info -lLLVMR600AsmPrinter -lLLVMSystemZDisassembler -lLLVMSystemZCodeGen -lLLVMSystemZAsmParser -lLLVMSystemZDesc -lLLVMSystemZInfo -lLLVMSystemZAsmPrinter -lLLVMHexagonCodeGen -lLLVMHexagonAsmPrinter -lLLVMHexagonDesc -lLLVMHexagonInfo -lLLVMNVPTXCodeGen -lLLVMNVPTXDesc -lLLVMNVPTXInfo -lLLVMNVPTXAsmPrinter -lLLVMCppBackendCodeGen -lLLVMCppBackendInfo -lLLVMMSP430CodeGen -lLLVMMSP430Desc -lLLVMMSP430Info -lLLVMMSP430AsmPrinter -lLLVMXCoreDisassembler -lLLVMXCoreCodeGen -lLLVMXCoreDesc -lLLVMXCoreInfo -lLLVMXCoreAsmPrinter -lLLVMMipsDisassembler -lLLVMMipsCodeGen -lLLVMMipsAsmParser -lLLVMMipsDesc -lLLVMMipsInfo -lLLVMMipsAsmPrinter -lLLVMARMDisassembler -lLLVMARMCodeGen -lLLVMARMAsmParser -lLLVMARMDesc -lLLVMARMInfo -lLLVMARMAsmPrinter -lLLVMAArch64Disassembler -lLLVMAArch64CodeGen -lLLVMAArch64AsmParser -lLLVMAArch64Desc -lLLVMAArch64Info -lLLVMAArch64AsmPrinter -lLLVMAArch64Utils -lLLVMPowerPCCodeGen -lLLVMPowerPCAsmParser -lLLVMPowerPCDesc -lLLVMPowerPCInfo -lLLVMPowerPCAsmPrinter -lLLVMSparcCodeGen -lLLVMSparcDesc -lLLVMSparcInfo -lLLVMX86Disassembler -lLLVMX86AsmParser -lLLVMX86CodeGen -lLLVMSelectionDAG -lLLVMAsmPrinter -lLLVMX86Desc -lLLVMX86Info -lLLVMX86AsmPrinter -lLLVMX86Utils -lLLVMMCDisassembler -lLLVMMCParser -lLLVMInterpreter -lLLVMMCJIT -lLLVMJIT -lLLVMCodeGen -lLLVMObjCARCOpts -lLLVMScalarOpts -lLLVMInstCombine -lLLVMTransformUtils -lLLVMipa -lLLVMAnalysis -lLLVMRuntimeDyld -lLLVMExecutionEngine -lLLVMTarget -lLLVMMC -lLLVMObject -lLLVMCore -lLLVMSupport -ltinfo +endif # The backend doesn't need everything that's in LIBS, however LIBS := $(filter-out -lreadline -ledit -ltermcap -lncurses -lcurses, $(LIBS)) @@ -432,8 +434,10 @@ endif cp -d $(LZ4_LIB_PATH)/liblz4* '$(DESTDIR)$(libdir)/' cp -d $(XGBOOST_LIB_PATH)/libxgboost* '$(DESTDIR)$(libdir)/' cp -d $(CJSON_LIB_PATH)/libcjson* '$(DESTDIR)$(libdir)/' +ifneq ($(host_cpu), loogarch64) cp $(PLJAVA_LIB_PATH)/* '$(DESTDIR)$(libdir)/' cp $(PLJAVA_JAR_PATH)/$(JARPLJAVA) '$(DESTDIR)$(pkglibdir)/java/' +endif cp -r $(top_builddir)/../$(BUILD_TOOLS_PATH)/jdk8/jdk1.8.0_77/jre/* '$(DESTDIR)$(bindir)/../jre/' install-bin: gaussdb $(POSTGRES_IMP) installdirs libcgroup @@ -493,7 +497,9 @@ uninstall: rm -f '$(DESTDIR)$(bindir)/gaussdb$(X)' '$(DESTDIR)$(bindir)/gs_encrypt' rm -f $(DESTDIR)$(libdir)/libcgroup* rm -f $(DESTDIR)$(libdir)/libsimsearch/* +ifneq ($(host_cpu), loongarch64) rm -f '$(DESTDIR)$(libdir)/$(LIBPLJAVA)' '$(DESTDIR)$(pkglibdir)/java/$(JARPLJAVA)' +endif ifeq ($(MAKE_EXPORTS), true) rm -f '$(DESTDIR)$(pkglibdir)/$(POSTGRES_IMP)' diff --git a/src/gausskernel/cbb/CMakeLists.txt b/src/gausskernel/cbb/CMakeLists.txt index 1ac64307e..98db5faa8 100755 --- a/src/gausskernel/cbb/CMakeLists.txt +++ b/src/gausskernel/cbb/CMakeLists.txt @@ -14,7 +14,11 @@ set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/workload ) -add_subdirectory(bbox) +if("${ENABLE_BBOX}" STREQUAL "ON") + add_subdirectory(bbox) +else() + list(REMOVE_ITEM CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/bbox) +endif() add_subdirectory(instruments) add_subdirectory(communication) add_subdirectory(extension) diff --git a/src/gausskernel/cbb/Makefile b/src/gausskernel/cbb/Makefile index df707a242..a36139a0b 100644 --- a/src/gausskernel/cbb/Makefile +++ b/src/gausskernel/cbb/Makefile @@ -24,6 +24,9 @@ subdir = src/gausskernel/cbb top_builddir = ../../.. include $(top_builddir)/src/Makefile.global -SUBDIRS = extension grpc instruments communication workload utils bbox +SUBDIRS = extension grpc instruments communication workload utils +ifeq ($(enable_bbox), yes) +SUBDIRS += bbox +endif include $(top_srcdir)/src/gausskernel/common.mk diff --git a/src/gausskernel/process/postmaster/pagewriter.cpp b/src/gausskernel/process/postmaster/pagewriter.cpp index 36f4501d4..29152520e 100755 --- a/src/gausskernel/process/postmaster/pagewriter.cpp +++ b/src/gausskernel/process/postmaster/pagewriter.cpp @@ -426,6 +426,7 @@ bool is_dirty_page_queue_full(BufferDesc* buf) return false; } +#if (defined(__x86_64__) || defined(__aarch64__)) && !defined(__USE_SPINLOCK) bool atomic_push_pending_flush_queue(XLogRecPtr* queue_head_lsn, uint64* new_tail_loc) { uint128_u compare; @@ -458,6 +459,7 @@ bool atomic_push_pending_flush_queue(XLogRecPtr* queue_head_lsn, uint64* new_tai *new_tail_loc -= 1; return true; } +#endif bool push_pending_flush_queue(Buffer buffer) { @@ -465,10 +467,10 @@ bool push_pending_flush_queue(Buffer buffer) uint64 actual_loc; XLogRecPtr queue_head_lsn = InvalidXLogRecPtr; BufferDesc* buf_desc = GetBufferDescriptor(buffer - 1); - bool push_finish = false; Assert(XLogRecPtrIsInvalid(pg_atomic_read_u64(&buf_desc->extra->rec_lsn))); -#if defined(__x86_64__) || defined(__aarch64__) +#if defined(__x86_64__) || defined(__aarch64__) && !defined(__USE_SPINLOCK) + bool push_finish = false; push_finish = atomic_push_pending_flush_queue(&queue_head_lsn, &new_tail_loc); if (!push_finish) { return false; @@ -476,11 +478,15 @@ bool push_pending_flush_queue(Buffer buffer) #else SpinLockAcquire(&g_instance.ckpt_cxt_ctl->queue_lock); - if ((uint64)(get_dirty_page_num() + PAGE_QUEUE_SLOT_MIN_RESERVE_NUM) >= + if ((uint64)(g_instance.ckpt_cxt_ctl->dirty_page_queue_tail + - g_instance.ckpt_cxt_ctl->dirty_page_queue_head + + PAGE_QUEUE_SLOT_MIN_RESERVE_NUM) >= g_instance.ckpt_cxt_ctl->dirty_page_queue_size) { SpinLockRelease(&g_instance.ckpt_cxt_ctl->queue_lock); return false; } + + queue_head_lsn = g_instance.ckpt_cxt_ctl->dirty_page_queue_reclsn; new_tail_loc = g_instance.ckpt_cxt_ctl->dirty_page_queue_tail; g_instance.ckpt_cxt_ctl->dirty_page_queue_tail++; SpinLockRelease(&g_instance.ckpt_cxt_ctl->queue_lock); @@ -509,7 +515,7 @@ uint64 get_dirty_page_queue_tail() { uint64 tail = 0; -#if defined(__x86_64__) || defined(__aarch64__) +#if defined(__x86_64__) || defined(__aarch64__) && !defined(__USE_SPINLOCK) tail = pg_atomic_barrier_read_u64(&g_instance.ckpt_cxt_ctl->dirty_page_queue_tail); #else SpinLockAcquire(&g_instance.ckpt_cxt_ctl->queue_lock); diff --git a/src/gausskernel/process/postmaster/postmaster.cpp b/src/gausskernel/process/postmaster/postmaster.cpp index f3918b22c..85e77d64a 100644 --- a/src/gausskernel/process/postmaster/postmaster.cpp +++ b/src/gausskernel/process/postmaster/postmaster.cpp @@ -66,7 +66,9 @@ */ #include "postgres.h" #include "knl/knl_variable.h" +#ifdef ENABLE_BBOX #include "gs_bbox.h" +#endif #include #include #include @@ -342,7 +344,9 @@ pthread_rwlock_t hba_rwlock = PTHREAD_RWLOCK_INITIALIZER; extern bool data_catchup; extern bool wal_catchup; +#ifdef ENABLE_BBOX char g_bbox_dump_path[1024] = {0}; +#endif #define CHECK_FOR_PROCDIEPENDING() \ do { \ @@ -2219,13 +2223,17 @@ int PostmasterMain(int argc, char* argv[]) if (strlen(GetConfigOption(const_cast("unix_socket_directory"), true, false)) != 0) { PythonFencedMasterModel = true; +#ifdef ENABLE_BBOX /* disable bbox for fenced UDF process */ SetConfigOption("enable_bbox_dump", "false", PGC_POSTMASTER, PGC_S_ARGV); +#endif } #else if (FencedUDFMasterMode) { +#ifdef ENABLE_BBOX /* disable bbox for fenced UDF process */ SetConfigOption("enable_bbox_dump", "false", PGC_POSTMASTER, PGC_S_ARGV); +#endif } else if (!SelectConfigFiles(userDoption, progname)) { ExitPostmaster(1); } @@ -2851,8 +2859,10 @@ int PostmasterMain(int argc, char* argv[]) (void)gspqsignal(SIGXFSZ, SIG_IGN); /* ignored */ #endif +#ifdef ENABLE_BBOX /* core dump injection */ bbox_initialize(); +#endif /* * Initialize stats collection subsystem (this does NOT start the diff --git a/src/gausskernel/runtime/Makefile b/src/gausskernel/runtime/Makefile index 20138113e..95414a266 100644 --- a/src/gausskernel/runtime/Makefile +++ b/src/gausskernel/runtime/Makefile @@ -11,7 +11,9 @@ include $(top_builddir)/src/Makefile.global SUBDIRS = executor opfusion vecexecutor ifneq ($(enable_lite_mode), yes) -SUBDIRS += codegen +ifeq ($(enable_llvm), yes) + SUBDIRS += codegen +endif endif include $(top_srcdir)/src/gausskernel/common.mk diff --git a/src/gausskernel/runtime/executor/instrument.cpp b/src/gausskernel/runtime/executor/instrument.cpp index f912091a5..a18d89f4f 100644 --- a/src/gausskernel/runtime/executor/instrument.cpp +++ b/src/gausskernel/runtime/executor/instrument.cpp @@ -87,12 +87,14 @@ static inline uint64 rdtsc(void) asm volatile("isb; mrs %0, cntvct_el0" : "=r"(cval) : : "memory"); return cval; -#else +#elif defined(__GNUC__) && (defined(__x86_64__) || defined(__i386__)) uint32 hi = 0; uint32 lo = 0; asm volatile("rdtsc" : "=a"(lo), "=d"(hi)); return ((uint64)lo) | (((uint64)hi) << 32); +#else + return clock(); #endif } #else diff --git a/src/gausskernel/runtime/vecexecutor/vectorsonic/vsonichash.cpp b/src/gausskernel/runtime/vecexecutor/vectorsonic/vsonichash.cpp index 312f8e556..4c1d2e95b 100644 --- a/src/gausskernel/runtime/vecexecutor/vectorsonic/vsonichash.cpp +++ b/src/gausskernel/runtime/vecexecutor/vectorsonic/vsonichash.cpp @@ -30,8 +30,10 @@ #include "utils/dynahash.h" #ifdef __aarch64__ #include -#else +#elif defined(__GNUC__) && (defined(__x86_64__) || defined(__i386__)) #include +#else +#include "port/pg_crc32c.h" #endif extern bool anls_opt_is_on(AnalysisOpt dfx_opt); @@ -42,8 +44,15 @@ extern bool anls_opt_is_on(AnalysisOpt dfx_opt); #ifdef __aarch64__ #define HASH_INT32_CRC(c, k) __crc32cw(c, k) -#else +#elif defined(__GNUC__) && (defined(__x86_64__) || defined(__i386__)) #define HASH_INT32_CRC(c, k) _mm_crc32_u32(c, k) +#else +FORCE_INLINE +uint32 __crc32cw_normal(uint32 seed, int32 key){ + return pg_comp_crc32c_sb8(seed, (const unsigned char *)&key, 4); +} + +#define HASH_INT32_CRC(c, k) __crc32cw_normal(c, k) #endif #define HASH_CRC_SEED 0xFFFFFFFF @@ -53,12 +62,12 @@ extern Datum hash_bi_key(Numeric key); FORCE_INLINE uint32 hashquickany(uint32 seed, register const unsigned char* data, register int len) { - const unsigned char* p = data; - const unsigned char* pend = p + len; - uint32 crc = seed; #ifdef __aarch64__ + const unsigned char* p = data; + const unsigned char* pend = p + len; + while (p + 8 <= pend) { crc = (uint32)__crc32d(crc, *((const uint64*)p)); p += 8; @@ -75,7 +84,9 @@ uint32 hashquickany(uint32 seed, register const unsigned char* data, register in crc = __crc32cb(crc, *p); p++; } -#else +#elif defined(__GNUC__) && (defined(__x86_64__) || defined(__i386__)) + const unsigned char* p = data; + const unsigned char* pend = p + len; while (p + 8 <= pend) { crc = (uint32)_mm_crc32_u64(crc, *((const uint64*)p)); p += 8; @@ -92,6 +103,8 @@ uint32 hashquickany(uint32 seed, register const unsigned char* data, register in crc = _mm_crc32_u8(crc, *p); p++; } +#else + crc = pg_comp_crc32c_sb8(seed, data, len); #endif return crc; } diff --git a/src/gausskernel/storage/access/transam/double_write.cpp b/src/gausskernel/storage/access/transam/double_write.cpp index 4cc88c68a..d60acd773 100644 --- a/src/gausskernel/storage/access/transam/double_write.cpp +++ b/src/gausskernel/storage/access/transam/double_write.cpp @@ -36,7 +36,9 @@ #include "utils/palloc.h" #include "gstrace/gstrace_infra.h" #include "gstrace/access_gstrace.h" +#ifdef ENABLE_BBOX #include "gs_bbox.h" +#endif #include "postmaster/bgwriter.h" #include "knl/knl_thread.h" #include "tde_key_management/tde_key_storage.h" @@ -1539,9 +1541,12 @@ static void dw_file_cxt_init_batch(int id, dw_batch_file_context *batch_file_cxt (void)dw_recover_batch_file_head(batch_file_cxt); batch_file_cxt->buf = buf; + +#ifdef ENABLE_BBOX if (BBOX_BLACKLIST_DW_BUFFER) { bbox_blacklist_add(DW_BUFFER, buf, buf_size - BLCKSZ - BLCKSZ); } +#endif batch_file_cxt->write_pos = 0; batch_file_cxt->flush_page = 0; diff --git a/src/gausskernel/storage/access/transam/extreme_rto/dispatcher.cpp b/src/gausskernel/storage/access/transam/extreme_rto/dispatcher.cpp index eb77095e0..8a5256878 100755 --- a/src/gausskernel/storage/access/transam/extreme_rto/dispatcher.cpp +++ b/src/gausskernel/storage/access/transam/extreme_rto/dispatcher.cpp @@ -1634,7 +1634,7 @@ void ItemBlocksOfItemIsReplayed(RedoItem *item) void GetLsnCheckInfo(uint64 *curPosition, XLogRecPtr *curLsn) { volatile LsnCheckCtl *checkCtl = g_dispatcher->lsnCheckCtl; -#if defined(__x86_64__) || defined(__aarch64__) +#if defined(__x86_64__) || defined(__aarch64__) && !defined(__USE_SPINLOCK) uint128_u current = atomic_compare_and_swap_u128((uint128_u *)&checkCtl->curPosition); Assert(sizeof(checkCtl->curPosition) == sizeof(uint64)); Assert(sizeof(checkCtl->curLsn) == sizeof(XLogRecPtr)); @@ -1652,7 +1652,7 @@ void GetLsnCheckInfo(uint64 *curPosition, XLogRecPtr *curLsn) void SetLsnCheckInfo(uint64 curPosition, XLogRecPtr curLsn) { volatile LsnCheckCtl *checkCtl = g_dispatcher->lsnCheckCtl; -#if defined(__x86_64__) || defined(__aarch64__) +#if defined(__x86_64__) || defined(__aarch64__) && !defined(__USE_SPINLOCK) uint128_u exchange; uint128_u compare = atomic_compare_and_swap_u128((uint128_u *)&checkCtl->curPosition); diff --git a/src/gausskernel/storage/access/transam/extreme_rto/page_redo.cpp b/src/gausskernel/storage/access/transam/extreme_rto/page_redo.cpp index 1b2db5d13..73dce2def 100755 --- a/src/gausskernel/storage/access/transam/extreme_rto/page_redo.cpp +++ b/src/gausskernel/storage/access/transam/extreme_rto/page_redo.cpp @@ -263,7 +263,7 @@ void DestroyPageRedoWorker(PageRedoWorker *worker) void SetCompletedReadEndPtr(PageRedoWorker *worker, XLogRecPtr readPtr, XLogRecPtr endPtr) { volatile PageRedoWorker *tmpWk = worker; -#if defined(__x86_64__) || defined(__aarch64__) +#if defined(__x86_64__) || defined(__aarch64__) && !defined(__USE_SPINLOCK) uint128_u exchange; uint128_u current; uint128_u compare = atomic_compare_and_swap_u128((uint128_u *)&tmpWk->lastReplayedReadRecPtr); @@ -292,7 +292,7 @@ void SetCompletedReadEndPtr(PageRedoWorker *worker, XLogRecPtr readPtr, XLogRecP void GetCompletedReadEndPtr(PageRedoWorker *worker, XLogRecPtr *readPtr, XLogRecPtr *endPtr) { volatile PageRedoWorker *tmpWk = worker; -#if defined(__x86_64__) || defined(__aarch64__) +#if defined(__x86_64__) || defined(__aarch64__) && !defined(__USE_SPINLOCK) uint128_u compare = atomic_compare_and_swap_u128((uint128_u *)&tmpWk->lastReplayedReadRecPtr); Assert(sizeof(tmpWk->lastReplayedReadRecPtr) == 8); Assert(sizeof(tmpWk->lastReplayedEndRecPtr) == 8); diff --git a/src/gausskernel/storage/access/transam/xlog.cpp b/src/gausskernel/storage/access/transam/xlog.cpp index 6569d30cf..211c97d20 100755 --- a/src/gausskernel/storage/access/transam/xlog.cpp +++ b/src/gausskernel/storage/access/transam/xlog.cpp @@ -65,7 +65,9 @@ #include "catalog/pg_database.h" #include "catalog/storage.h" #include "catalog/storage_xlog.h" +#ifdef ENABLE_BBOX #include "gs_bbox.h" +#endif #include "libpq/pqsignal.h" #include "miscadmin.h" #ifdef PGXC @@ -884,7 +886,7 @@ static void ReserveXLogInsertByteLocation(uint32 size, uint32 lastRecordSize, ui * because the usable byte position doesn't include any headers, reserving * X bytes from WAL is almost as simple as "CurrBytePos += X". */ -#if defined(__x86_64__) || defined(__aarch64__) +#if defined(__x86_64__) || defined(__aarch64__) && !defined(__USE_SPINLOCK) union Union128 compare; union Union128 exchange; union Union128 current; @@ -922,25 +924,31 @@ static void ReserveXLogInsertByteLocation(uint32 size, uint32 lastRecordSize, ui UINT128_COPY(compare.value, current.value); goto loop; } + + *currlrc_ptr = compare.struct128.LRC; + *StartBytePos = compare.struct128.currentBytePos; + *EndBytePos = exchange.struct128.currentBytePos; + *PrevBytePos = compare.struct128.currentBytePos - compare.struct128.byteSize; #else - uint64 startbytepos; - uint64 endbytepos; - uint64 prevbytepos; - +loop1: SpinLockAcquire(&Insert->insertpos_lck); - startbytepos = Insert->CurrBytePos; - endbytepos = startbytepos + size; - prevbytepos = Insert->PrevBytePos; - Insert->CurrBytePos = endbytepos; - Insert->PrevBytePos = endbytepos - lastRecordSize; + if (unlikely(Insert->CurrLRC == WAL_COPY_SUSPEND)) { + SpinLockRelease(&Insert->insertpos_lck); + goto loop1; + } + + *currlrc_ptr = Insert->CurrLRC; + *StartBytePos = Insert->CurrBytePos; + *EndBytePos = Insert->CurrBytePos + size; + *PrevBytePos = Insert->CurrBytePos - Insert->PrevByteSize; + Insert->CurrLRC = (Insert->CurrLRC + 1) & 0x7FFFFFFF; + Insert->PrevByteSize = lastRecordSize; + Insert->CurrBytePos = *EndBytePos; SpinLockRelease(&Insert->insertpos_lck); #endif /* __x86_64__ || __aarch64__ */ - *currlrc_ptr = compare.struct128.LRC; - *StartBytePos = compare.struct128.currentBytePos; - *EndBytePos = exchange.struct128.currentBytePos; - *PrevBytePos = compare.struct128.currentBytePos - compare.struct128.byteSize; + } /* @@ -1401,7 +1409,7 @@ static void ReserveXLogInsertLocation(uint32 size, XLogRecPtr *StartPos, XLogRec * because the usable byte position doesn't include any headers, reserving * X bytes from WAL is almost as simple as "CurrBytePos += X". */ -#if defined(__x86_64__) || defined(__aarch64__) +#if defined(__x86_64__) || defined(__aarch64__) && !defined(__USE_SPINLOCK) union Union128 compare; union Union128 exchange; union Union128 current; @@ -1440,25 +1448,29 @@ static void ReserveXLogInsertLocation(uint32 size, XLogRecPtr *StartPos, XLogRec goto loop1; } -#else - uint64 startbytepos; - uint64 endbytepos; - uint64 prevbytepos; + *currlrc_ptr = compare.struct128.LRC; + *StartPos = XLogBytePosToRecPtr(compare.struct128.currentBytePos); + *EndPos = XLogBytePosToEndRecPtr(exchange.struct128.currentBytePos); + *PrevPtr = XLogBytePosToRecPtr(compare.struct128.currentBytePos - compare.struct128.byteSize); +#else +loop1: SpinLockAcquire(&Insert->insertpos_lck); - - startbytepos = Insert->CurrBytePos; - prevbytepos = Insert->PrevBytePos; - endbytepos = startbytepos + size; - Insert->CurrBytePos = endbytepos; - Insert->PrevBytePos = startbytepos; + if (unlikely(Insert->CurrLRC == WAL_COPY_SUSPEND)) { + SpinLockRelease(&Insert->insertpos_lck); + pg_usleep(1); + goto loop1; + } + *currlrc_ptr = Insert->CurrLRC; + *PrevPtr = XLogBytePosToRecPtr(Insert->CurrBytePos - Insert->PrevByteSize); + *StartPos = XLogBytePosToRecPtr(Insert->CurrBytePos); + Insert->CurrBytePos = Insert->CurrBytePos + size; + *EndPos = XLogBytePosToEndRecPtr(Insert->CurrBytePos); + Insert->PrevByteSize = size; + Insert->CurrLRC = (Insert->CurrLRC + 1) & 0x7FFFFFFF; SpinLockRelease(&Insert->insertpos_lck); #endif /* __x86_64__|| __aarch64__ */ - *currlrc_ptr = compare.struct128.LRC; - *StartPos = XLogBytePosToRecPtr(compare.struct128.currentBytePos); - *EndPos = XLogBytePosToEndRecPtr(exchange.struct128.currentBytePos); - *PrevPtr = XLogBytePosToRecPtr(compare.struct128.currentBytePos - compare.struct128.byteSize); } /* @@ -1487,7 +1499,7 @@ static bool ReserveXLogSwitch(XLogRecPtr *StartPos, XLogRecPtr *EndPos, XLogRecP * are no other inserters competing for it. GetXLogInsertRecPtr() does * compete for it, but that's not called very frequently. */ -#if defined(__x86_64__) || defined(__aarch64__) +#if defined(__x86_64__) || defined(__aarch64__) && !defined(__USE_SPINLOCK) uint128_u exchange; uint128_u current; uint128_u compare = atomic_compare_and_swap_u128((uint128_u *)&Insert->CurrBytePos); @@ -1538,7 +1550,8 @@ static bool ReserveXLogSwitch(XLogRecPtr *StartPos, XLogRecPtr *EndPos, XLogRecP } endbytepos = startbytepos + size; - prevbytesize = startbytepos - Insert->PrevByteSize; + prevbytesize = Insert->PrevByteSize; + currlrc = Insert->CurrLRC; *StartPos = XLogBytePosToRecPtr(startbytepos); *EndPos = XLogBytePosToEndRecPtr(endbytepos); @@ -1550,7 +1563,8 @@ static bool ReserveXLogSwitch(XLogRecPtr *StartPos, XLogRecPtr *EndPos, XLogRecP endbytepos = XLogRecPtrToBytePos(*EndPos); } Insert->CurrBytePos = endbytepos; - Insert->PrevByteSize = size; + Insert->PrevByteSize = (uint32)(endbytepos - startbytepos); + Insert->CurrLRC = currlrc; SpinLockRelease(&Insert->insertpos_lck); #endif /* __x86_64__ || __aarch64__ */ @@ -1568,11 +1582,12 @@ static bool ReserveXLogSwitch(XLogRecPtr *StartPos, XLogRecPtr *EndPos, XLogRecP static void StartSuspendWalInsert(int32 *const lastlrc_ptr) { volatile XLogCtlInsert *Insert = &t_thrd.shemem_ptr_cxt.XLogCtl->Insert; - uint64 startbytepos; - uint32 prevbytesize; int32 currlrc; int32 ientry; volatile WALInsertStatusEntry *entry; +#if (defined(__x86_64__) || defined(__aarch64__)) && !defined(__USE_SPINLOCK) + uint64 startbytepos; + uint32 prevbytesize; uint128_u compare; uint128_u exchange; uint128_u current; @@ -1621,7 +1636,25 @@ static void StartSuspendWalInsert(int32 *const lastlrc_ptr) UINT128_COPY(compare, current); goto loop; } +#else +loop: + SpinLockAcquire(&Insert->insertpos_lck); + currlrc = Insert->CurrLRC; + if (unlikely(currlrc == WAL_COPY_SUSPEND)) { + SpinLockRelease(&Insert->insertpos_lck); + goto loop; + } + + /* + * Return the last LRC + */ + *lastlrc_ptr = currlrc; + Insert->CurrLRC = WAL_COPY_SUSPEND; + + SpinLockRelease(&Insert->insertpos_lck); +#endif /* __x86_64__ || __aarch64__ */ + /* /* * Wait for the WAL copy thread obtaining the "*lastlrc_ptr - 1" to finish * *lastlrc_ptr is the LRC for the next WAL insert thread to acquire @@ -6439,10 +6472,12 @@ void XLOGShmemInit(void) /* The memory of the memset sometimes exceeds 2 GB. so, memset_s cannot be used. */ MemSet(t_thrd.shemem_ptr_cxt.XLogCtl->pages, 0, (Size)XLOG_BLCKSZ * g_instance.attr.attr_storage.XLOGbuffers); +#ifdef ENABLE_BBOX if (BBOX_BLACKLIST_XLOG_BUFFER) { bbox_blacklist_add(XLOG_BUFFER, t_thrd.shemem_ptr_cxt.XLogCtl->pages, (uint64)XLOG_BLCKSZ * g_instance.attr.attr_storage.XLOGbuffers); } +#endif /* * Do basic initialization of XLogCtl shared data. (StartupXLOG will fill @@ -15226,7 +15261,9 @@ XLogRecPtr do_roach_stop_backup(const char *backupidstr) { bool backup_started_in_recovery = false; XLogCtlInsert *Insert = &t_thrd.shemem_ptr_cxt.XLogCtl->Insert; +#if defined(__aarch64__) || defined(__x86_64__) || defined(__i386__) uint64 current_bytepos; +#endif XLogRecPtr stoppoint; char backup_label[MAXPGPATH]; char backup_label_done[MAXPGPATH]; @@ -15259,7 +15296,7 @@ XLogRecPtr do_roach_stop_backup(const char *backupidstr) stoppoint = t_thrd.shemem_ptr_cxt.ControlFile->minRecoveryPoint; LWLockRelease(ControlFileLock); } else { -#if defined(__x86_64__) || defined(__aarch64__) +#if defined(__x86_64__) || defined(__aarch64__) && !defined(__USE_SPINLOCK) current_bytepos = pg_atomic_barrier_read_u64((uint64 *)&Insert->CurrBytePos); stoppoint = XLogBytePosToEndRecPtr(current_bytepos); #else @@ -15511,7 +15548,7 @@ XLogRecPtr enable_delay_ddl_recycle(void) LWLockAcquire(CBMParseXlogLock, LW_EXCLUSIVE); -#if defined(__x86_64__) || defined(__aarch64__) +#if defined(__x86_64__) || defined(__aarch64__) && !defined(__USE_SPINLOCK) uint64 current_bytepos = pg_atomic_barrier_read_u64((uint64 *)&Insert->CurrBytePos); curDelayRange.startLSN = XLogBytePosToEndRecPtr(current_bytepos); #else @@ -15645,7 +15682,7 @@ void disable_delay_ddl_recycle(XLogRecPtr barrierLSN, bool isForce, XLogRecPtr * */ LWLockAcquire(RelfilenodeReuseLock, LW_EXCLUSIVE); -#if defined(__x86_64__) || defined(__aarch64__) +#if defined(__x86_64__) || defined(__aarch64__) && !defined(__USE_SPINLOCK) uint64 current_bytepos = pg_atomic_barrier_read_u64((uint64 *)&Insert->CurrBytePos); delayRange.endLSN = XLogBytePosToEndRecPtr(current_bytepos); #else @@ -15733,7 +15770,7 @@ XLogRecPtr enable_delay_ddl_recycle_with_slot(const char *slotname) /* hold this lock to push cbm parse exact to the ddl stop position */ LWLockAcquire(CBMParseXlogLock, LW_EXCLUSIVE); -#if defined(__x86_64__) || defined(__aarch64__) +#if defined(__x86_64__) || defined(__aarch64__) && !defined(__USE_SPINLOCK) uint64 current_bytepos = pg_atomic_barrier_read_u64((uint64 *)&Insert->CurrBytePos); delay_start_lsn = XLogBytePosToEndRecPtr(current_bytepos); #else @@ -15805,7 +15842,7 @@ void disable_delay_ddl_recycle_with_slot(const char *slotname, XLogRecPtr *start */ LWLockAcquire(RelfilenodeReuseLock, LW_EXCLUSIVE); -#if defined(__x86_64__) || defined(__aarch64__) +#if defined(__x86_64__) || defined(__aarch64__) && !defined(__USE_SPINLOCK) uint64 current_bytepos = pg_atomic_barrier_read_u64((uint64 *)&Insert->CurrBytePos); delay_end_lsn = XLogBytePosToEndRecPtr(current_bytepos); #else @@ -16244,7 +16281,7 @@ XLogRecPtr GetXLogInsertRecPtr(void) volatile XLogCtlInsert *Insert = &t_thrd.shemem_ptr_cxt.XLogCtl->Insert; uint64 current_bytepos; -#if defined(__x86_64__) || defined(__aarch64__) +#if defined(__x86_64__) || defined(__aarch64__) && !defined(__USE_SPINLOCK) current_bytepos = pg_atomic_barrier_read_u64((uint64 *)&Insert->CurrBytePos); #else SpinLockAcquire(&Insert->insertpos_lck); @@ -16266,7 +16303,7 @@ XLogRecPtr GetXLogInsertEndRecPtr(void) volatile XLogCtlInsert *Insert = &t_thrd.shemem_ptr_cxt.XLogCtl->Insert; uint64 current_bytepos; -#if defined(__x86_64__) || defined(__aarch64__) +#if defined(__x86_64__) || defined(__aarch64__) && !defined(__USE_SPINLOCK) current_bytepos = pg_atomic_barrier_read_u64((uint64 *)&Insert->CurrBytePos); #else SpinLockAcquire(&Insert->insertpos_lck); @@ -18374,7 +18411,7 @@ bool IsRoachRestore(void) } const uint UPDATE_REC_XLOG_NUM = 4; -#if defined(__x86_64__) || defined(__aarch64__) +#if defined(__x86_64__) || defined(__aarch64__) && !defined(__USE_SPINLOCK) bool atomic_update_dirty_page_queue_rec_lsn(XLogRecPtr current_insert_lsn, bool need_immediately_update) { XLogRecPtr cur_rec_lsn = InvalidXLogRecPtr; @@ -18431,7 +18468,7 @@ void update_dirty_page_queue_rec_lsn(XLogRecPtr current_insert_lsn, bool need_im } } -#if defined(__x86_64__) || defined(__aarch64__) +#if defined(__x86_64__) || defined(__aarch64__) && !defined(__USE_SPINLOCK) is_update = atomic_update_dirty_page_queue_rec_lsn(current_insert_lsn, need_immediately_update); #else SpinLockAcquire(&g_instance.ckpt_cxt_ctl->queue_lock); @@ -18451,7 +18488,7 @@ void update_dirty_page_queue_rec_lsn(XLogRecPtr current_insert_lsn, bool need_im uint64 get_dirty_page_queue_rec_lsn() { uint64 dirty_page_queue_rec_lsn = 0; -#if defined(__x86_64__) || defined(__aarch64__) +#if defined(__x86_64__) || defined(__aarch64__) && !defined(__USE_SPINLOCK) dirty_page_queue_rec_lsn = pg_atomic_barrier_read_u64(&g_instance.ckpt_cxt_ctl->dirty_page_queue_reclsn); #else SpinLockAcquire(&g_instance.ckpt_cxt_ctl->queue_lock); diff --git a/src/gausskernel/storage/buffer/buf_init.cpp b/src/gausskernel/storage/buffer/buf_init.cpp index 5a16ba890..42384e3e2 100644 --- a/src/gausskernel/storage/buffer/buf_init.cpp +++ b/src/gausskernel/storage/buffer/buf_init.cpp @@ -15,7 +15,9 @@ */ #include "postgres.h" #include "knl/knl_variable.h" +#ifdef ENABLE_BBOX #include "gs_bbox.h" +#endif #include "storage/buf/bufmgr.h" #include "storage/buf/buf_internals.h" #include "storage/nvm/nvm.h" @@ -108,10 +110,12 @@ void InitBufferPool(void) nvm_init(); } +#ifdef ENABLE_BBOX if (BBOX_BLACKLIST_SHARE_BUFFER) { /* Segment Buffer is exclued from the black list, as it contains many critical information for debug */ bbox_blacklist_add(SHARED_BUFFER, t_thrd.storage_cxt.BufferBlocks, NORMAL_SHARED_BUFFER_NUM * (Size)BLCKSZ); } +#endif /* * The array used to sort to-be-checkpointed buffer ids is located in diff --git a/src/gausskernel/storage/replication/dataqueue.cpp b/src/gausskernel/storage/replication/dataqueue.cpp index 5957abdd8..716b2e956 100644 --- a/src/gausskernel/storage/replication/dataqueue.cpp +++ b/src/gausskernel/storage/replication/dataqueue.cpp @@ -43,7 +43,9 @@ #include "storage/shmem.h" #include "storage/buf/bufmgr.h" #include "pgxc/pgxc.h" +#ifdef ENABLE_BBOX #include "gs_bbox.h" +#endif #define BCMElementArrayLen 8192 #define BCMElementArrayLenHalf (BCMElementArrayLen / 2) @@ -132,10 +134,11 @@ void DataWriterQueueShmemInit(void) if (foundDataQueue) { return; } - +#ifdef ENABLE_BBOX if (BBOX_BLACKLIST_DATA_WRITER_QUEUE) { bbox_blacklist_add(DATA_WRITER_QUEUE, t_thrd.dataqueue_cxt.DataWriterQueue, DataQueueShmemSize()); } +#endif rc = memset_s(t_thrd.dataqueue_cxt.DataWriterQueue, sizeof(DataQueueData), 0, sizeof(DataQueueData)); securec_check_c(rc, "", ""); diff --git a/src/gausskernel/storage/replication/datareceiver.cpp b/src/gausskernel/storage/replication/datareceiver.cpp index 83248fd4d..b79b6dc6f 100755 --- a/src/gausskernel/storage/replication/datareceiver.cpp +++ b/src/gausskernel/storage/replication/datareceiver.cpp @@ -68,7 +68,10 @@ #include "utils/ps_status.h" #include "utils/resowner.h" #include "utils/timestamp.h" +#ifdef ENABLE_BBOX #include "gs_bbox.h" +#endif + #ifdef ENABLE_UT #define static diff --git a/src/gausskernel/storage/replication/datasender.cpp b/src/gausskernel/storage/replication/datasender.cpp index 86131464c..1b136d5ed 100755 --- a/src/gausskernel/storage/replication/datasender.cpp +++ b/src/gausskernel/storage/replication/datasender.cpp @@ -71,7 +71,9 @@ #include "utils/resowner.h" #include "utils/timestamp.h" #include "gssignal/gs_signal.h" +#ifdef ENABLE_BBOX #include "gs_bbox.h" +#endif /* Flag indicates dummy are searching bcm files now */ static bool dummySearching; @@ -711,11 +713,13 @@ static int DataSndLoop(void) t_thrd.datasender_cxt.output_message = (char *)palloc(1 + sizeof(DataPageMessageHeader) + g_instance.attr.attr_storage.MaxSendSize * 1024); +#ifdef ENABLE_BBOX if (BBOX_BLACKLIST_DATA_MESSAGE_SEND) { bbox_blacklist_add(DATA_MESSAGE_SEND, t_thrd.datasender_cxt.output_message, (uint64)(1 + sizeof(DataPageMessageHeader) + g_instance.attr.attr_storage.MaxSendSize * 1024)); } +#endif /* * Allocate buffer that will be used for processing reply messages. As @@ -1128,10 +1132,11 @@ static void DataSndKill(int code, Datum arg) SpinLockRelease(&datasnd->mutex); dummySearching = false; +#ifdef ENABLE_BBOX if (BBOX_BLACKLIST_DATA_MESSAGE_SEND) { bbox_blacklist_remove(DATA_MESSAGE_SEND, t_thrd.datasender_cxt.output_message); } - +#endif ereport(LOG, (errmsg("datasender thread shut down"))); } diff --git a/src/gausskernel/storage/replication/walreceiver.cpp b/src/gausskernel/storage/replication/walreceiver.cpp index 34865ebeb..d8429622e 100755 --- a/src/gausskernel/storage/replication/walreceiver.cpp +++ b/src/gausskernel/storage/replication/walreceiver.cpp @@ -74,7 +74,9 @@ #include "utils/resowner.h" #include "utils/timestamp.h" #include "gssignal/gs_signal.h" +#ifdef ENABLE_BBOX #include "gs_bbox.h" +#endif #include "flock.h" #include "postmaster/postmaster.h" @@ -277,18 +279,22 @@ static void walRcvCtlBlockInit() securec_check_c(rc, "\0", "\0"); t_thrd.walreceiver_cxt.walRcvCtlBlock = (WalRcvCtlBlock *)buf; +#ifdef ENABLE_BBOX if (BBOX_BLACKLIST_WALREC_CTL_BLOCK) { bbox_blacklist_add(WALRECIVER_CTL_BLOCK, t_thrd.walreceiver_cxt.walRcvCtlBlock, len); } +#endif SpinLockInit(&t_thrd.walreceiver_cxt.walRcvCtlBlock->mutex); } static void walRcvCtlBlockFini() { +#ifdef ENABLE_BBOX if (BBOX_BLACKLIST_WALREC_CTL_BLOCK) { bbox_blacklist_remove(WALRECIVER_CTL_BLOCK, t_thrd.walreceiver_cxt.walRcvCtlBlock); } +#endif pfree(t_thrd.walreceiver_cxt.walRcvCtlBlock); t_thrd.walreceiver_cxt.walRcvCtlBlock = NULL; diff --git a/src/gausskernel/storage/replication/walsender.cpp b/src/gausskernel/storage/replication/walsender.cpp index 738d136e4..dd4a8d792 100755 --- a/src/gausskernel/storage/replication/walsender.cpp +++ b/src/gausskernel/storage/replication/walsender.cpp @@ -111,7 +111,9 @@ #include "postmaster/postmaster.h" #include "alarm/alarm.h" #include "utils/distribute_test.h" +#ifdef ENABLE_BBOX #include "gs_bbox.h" +#endif #include "lz4.h" #define InvalidPid ((ThreadId)(-1)) @@ -3823,10 +3825,12 @@ static void WSDataSendInit() if (!g_instance.attr.attr_storage.enable_mix_replication) { t_thrd.walsender_cxt.output_xlog_message = (char *)palloc(1 + sizeof(WalDataMessageHeader) + (int)WS_MAX_SEND_SIZE); +#ifdef ENABLE_BBOX if (BBOX_BLACKLIST_XLOG_MESSAGE_SEND) { bbox_blacklist_add(XLOG_MESSAGE_SEND, t_thrd.walsender_cxt.output_xlog_message, 1 + sizeof(WalDataMessageHeader) + (int)WS_MAX_SEND_SIZE); } +#endif } else { t_thrd.walsender_cxt.output_xlog_msg_prefix_len = 1 + sizeof(WalDataMessageHeader) + sizeof(uint32) + 1 + sizeof(XLogRecPtr); @@ -3842,10 +3846,12 @@ static void WSDataSendInit() t_thrd.walsender_cxt.wsXLogJustSendRegion->start_ptr = InvalidXLogRecPtr; t_thrd.walsender_cxt.wsXLogJustSendRegion->end_ptr = InvalidXLogRecPtr; +#ifdef ENABLE_BBOX if (BBOX_BLACKLIST_XLOG_MESSAGE_SEND) { bbox_blacklist_add(XLOG_MESSAGE_SEND, t_thrd.walsender_cxt.output_xlog_message, t_thrd.walsender_cxt.output_xlog_msg_prefix_len + (int)WS_MAX_SEND_SIZE); } +#endif } return; @@ -4657,9 +4663,11 @@ static void WalSndKill(int code, Datum arg) t_thrd.walsender_cxt.wsXLogJustSendRegion->start_ptr = InvalidXLogRecPtr; t_thrd.walsender_cxt.wsXLogJustSendRegion->end_ptr = InvalidXLogRecPtr; +#ifdef ENABLE_BBOX if (BBOX_BLACKLIST_XLOG_MESSAGE_SEND) { bbox_blacklist_remove(XLOG_MESSAGE_SEND, t_thrd.walsender_cxt.output_xlog_message); } +#endif if (XlogCopyStartPtr != InvalidXLogRecPtr) { LWLockAcquire(FullBuildXlogCopyStartPtrLock, LW_EXCLUSIVE); XlogCopyStartPtr = InvalidXLogRecPtr; diff --git a/src/include/access/extreme_rto/dispatcher.h b/src/include/access/extreme_rto/dispatcher.h index d6b8c8ea6..60b2d65b0 100644 --- a/src/include/access/extreme_rto/dispatcher.h +++ b/src/include/access/extreme_rto/dispatcher.h @@ -94,7 +94,7 @@ typedef struct RecordBufferAarray { typedef struct { uint64 curPosition; XLogRecPtr curLsn; -#if (!defined __x86_64__) && (!defined __aarch64__) +#if (!defined(__x86_64__) && !defined(__aarch64__)) || defined(__USE_SPINLOCK) /* protects lastReplayedReadRecPtr and lastReplayedEndRecPtr */ slock_t ptrLck; #endif diff --git a/src/include/access/extreme_rto/page_redo.h b/src/include/access/extreme_rto/page_redo.h index 90674a455..e05b6a606 100644 --- a/src/include/access/extreme_rto/page_redo.h +++ b/src/include/access/extreme_rto/page_redo.h @@ -77,7 +77,7 @@ struct PageRedoWorker { */ XLogRecPtr lastReplayedReadRecPtr; XLogRecPtr lastReplayedEndRecPtr; -#if (!defined __x86_64__) && (!defined __aarch64__) +#if (!defined(__x86_64__) && !defined(__aarch64__)) || defined(__USE_SPINLOCK) /* protects lastReplayedReadRecPtr and lastReplayedEndRecPtr */ slock_t ptrLck; #endif diff --git a/src/include/access/ondemand_extreme_rto/page_redo.h b/src/include/access/ondemand_extreme_rto/page_redo.h index 1615a3683..fe5f295e5 100644 --- a/src/include/access/ondemand_extreme_rto/page_redo.h +++ b/src/include/access/ondemand_extreme_rto/page_redo.h @@ -79,7 +79,7 @@ struct PageRedoWorker { */ XLogRecPtr lastReplayedReadRecPtr; XLogRecPtr lastReplayedEndRecPtr; -#if (!defined __x86_64__) && (!defined __aarch64__) +#if (!defined(__x86_64__) && !defined(__aarch64__)) || defined(__USE_SPINLOCK) /* protects lastReplayedReadRecPtr and lastReplayedEndRecPtr */ slock_t ptrLck; #endif diff --git a/src/include/access/parallel_recovery/page_redo.h b/src/include/access/parallel_recovery/page_redo.h index b1a4aadfb..a3103b0b8 100644 --- a/src/include/access/parallel_recovery/page_redo.h +++ b/src/include/access/parallel_recovery/page_redo.h @@ -73,7 +73,7 @@ struct PageRedoWorker { XLogRecPtr lastReplayedReadRecPtr; XLogRecPtr lastReplayedEndRecPtr; XLogRecPtr curReplayingReadRecPtr; -#if (!defined __x86_64__) && (!defined __aarch64__) +#if (!defined(__x86_64__) && !defined(__aarch64__)) || defined(__USE_SPINLOCK) /* protects lastReplayedReadRecPtr and lastReplayedEndRecPtr */ slock_t ptrLck; #endif diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h index f90e03ab4..49856efe6 100755 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@ -424,7 +424,7 @@ typedef struct XLogCtlInsert { uint32 PrevByteSize; int32 CurrLRC; -#if (!defined __x86_64__) && (!defined __aarch64__) +#if ((!defined __x86_64__) && (!defined __aarch64__)) || defined(__USE_SPINLOCK) slock_t insertpos_lck; /* protects CurrBytePos and PrevBytePos */ #endif /* diff --git a/src/include/communication/commproxy_interface.h b/src/include/communication/commproxy_interface.h index 67dab2f4a..9b999cca1 100644 --- a/src/include/communication/commproxy_interface.h +++ b/src/include/communication/commproxy_interface.h @@ -421,7 +421,9 @@ extern bool comm_compare_and_swap_32(volatile int32* dest, int32 oldval, int32 n #define gaussdb_numa_memory_unbind() #else -#define gaussdb_memory_barrier() +#define gaussdb_memory_barrier() __asm__ __volatile__("" ::: "memory") +#define gaussdb_read_barrier() gaussdb_memory_barrier() +#define gaussdb_write_barrier() gaussdb_memory_barrier() #define gaussdb_numa_memory_bind(i) #define gaussdb_numa_memory_unbind() diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 2419318ab..f0dbcf8ee 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -71,6 +71,14 @@ Supported only in single_node mode, not supported with (--enable-multiple-nodes) */ #undef ENABLE_MOT +/* Define to 1 if you want LLVM support. (--enable-llvm). + Supported only in single_node mode, not supported with (--enable-multiple-nodes) */ +#undef ENABLE_LLVM_COMPILE + +/* Define to 1 if you want BBOX support. (--enable-bbox). + Supported only in single_node mode, not supported with (--enable-multiple-nodes) */ +#undef ENABLE_BBOX + /* Define to 1 if you want to check memory. (--enable-memory-check) */ #undef ENABLE_MEMORY_CHECK diff --git a/src/include/storage/lock/s_lock.h b/src/include/storage/lock/s_lock.h index 6a64c9dfc..dae744fa1 100644 --- a/src/include/storage/lock/s_lock.h +++ b/src/include/storage/lock/s_lock.h @@ -606,6 +606,47 @@ static __inline__ int tas(volatile slock_t* lock) #endif /* __ns32k__ */ +#if defined(__loongarch__) /* loongarch */ +#define HAS_TEST_AND_SET + +typedef unsigned int slock_t; + +#define TAS(lock) tas(lock) + +static __inline__ int +tas(volatile slock_t *lock) +{ + register volatile slock_t *_l = lock; + register int _res; + register int _tmp; + + __asm__ __volatile__( + " ll.w %0, %2 \n" + " ori %1, %0, 1 \n" + " sc.w %1, %2 \n" + " xori %1, %1, 1 \n" + " or %0, %0, %1 \n" + " dbar 0 \n" +: "=&r" (_res), "=&r" (_tmp), "+R" (*_l) +: /* no inputs */ +: "memory"); + return _res; +} + +/* MIPS S_UNLOCK is almost standard but requires a "sync" instruction */ +#define S_UNLOCK(lock) \ +do \ +{ \ + __asm__ __volatile__( \ + " dbar 0 \n" \ +: /* no outputs */ \ +: /* no inputs */ \ +: "memory"); \ + *((volatile slock_t *) (lock)) = 0; \ +} while (0) + +#endif /* loongarch */ + #if defined(__alpha) || defined(__alpha__) /* Alpha */ /* * Correct multi-processor locking methods are explained in section 5.5.3 diff --git a/src/include/utils/atomic.h b/src/include/utils/atomic.h index 6adeb4ae9..edf6c70f6 100644 --- a/src/include/utils/atomic.h +++ b/src/include/utils/atomic.h @@ -553,6 +553,7 @@ static inline uint64 pg_atomic_barrier_read_u64(volatile uint64* ptr) * @IN newval: new value, should be 128bit(16bytes) aligned * @Return: old value */ +#if (defined(__x86_64__) || defined(__aarch64__)) && !defined(__USE_SPINLOCK) static inline uint128_u atomic_compare_and_swap_u128( volatile uint128_u* ptr, uint128_u oldval = uint128_u{0}, @@ -566,6 +567,7 @@ static inline uint128_u atomic_compare_and_swap_u128( return ret; #endif } +#endif /* (defined(__x86_64__) || defined(__aarch64__)) && !defined(__USE_SPINLOCK) */ #endif #endif /* ATOMIC_H */ From 9f6b440c31e6b3d0859e96d61cdb9c3b6b925e65 Mon Sep 17 00:00:00 2001 From: nexplorer-3e <60149113+nexplorer-3e@users.noreply.github.com> Date: Mon, 12 Feb 2024 18:20:02 +0800 Subject: [PATCH 302/304] add riscv support for cmake files TODO add support in makefile & comments --- build/script/cmake_package_mini.sh | 4 ++++ src/common/port/CMakeLists.txt | 8 ++++++++ src/include/storage/lock/s_lock.h | 4 ++-- src/lib/gstrace/CMakeLists.txt | 2 +- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/build/script/cmake_package_mini.sh b/build/script/cmake_package_mini.sh index 8c8e2e449..ca578b806 100644 --- a/build/script/cmake_package_mini.sh +++ b/build/script/cmake_package_mini.sh @@ -476,6 +476,10 @@ function install_gaussdb() CMAKE_OPT="$CMAKE_OPT -DENABLE_BBOX=OFF -DENABLE_JEMALLOC=OFF" fi + if [ "${PLATFORM_ARCH}"x == "riscv64"x ]; then + CMAKE_OPT="$CMAKE_OPT -DENABLE_BBOX=OFF" + fi + echo "CMAKE_OPT----> $CMAKE_OPT" echo "Begin run cmake for gaussdb server" >> "$LOG_FILE" 2>&1 echo "CMake options: ${CMAKE_OPT}" >> "$LOG_FILE" 2>&1 diff --git a/src/common/port/CMakeLists.txt b/src/common/port/CMakeLists.txt index b9a6a282a..e511eb0f5 100755 --- a/src/common/port/CMakeLists.txt +++ b/src/common/port/CMakeLists.txt @@ -94,6 +94,10 @@ if("${BUILD_TUPLE}" STREQUAL "loongarch64") list(REMOVE_ITEM TGT_port_SRC ${CMAKE_CURRENT_SOURCE_DIR}/pg_crc32c_choose.cpp ${CMAKE_CURRENT_SOURCE_DIR}/pg_crc32c_sse42.cpp) endif() +if("${BUILD_TUPLE}" STREQUAL "riscv64") + list(REMOVE_ITEM TGT_port_SRC ${CMAKE_CURRENT_SOURCE_DIR}/pg_crc32c_choose.cpp ${CMAKE_CURRENT_SOURCE_DIR}/pg_crc32c_sse42.cpp) +endif() + SET(TGT_pgport_INC ${PROJECT_SRC_DIR}/common/backend ${PROJECT_SRC_DIR}/common/port @@ -167,6 +171,10 @@ if("${BUILD_TUPLE}" STREQUAL "loongarch64") list(REMOVE_ITEM TGT_pgport_srv_SRC ${CMAKE_CURRENT_SOURCE_DIR}/port_srv/pg_crc32c_choose.cpp ${CMAKE_CURRENT_SOURCE_DIR}/port_srv/pg_crc32c_sse42.cpp) endif() +if("${BUILD_TUPLE}" STREQUAL "riscv64") + list(REMOVE_ITEM TGT_pgport_srv_SRC ${CMAKE_CURRENT_SOURCE_DIR}/port_srv/pg_crc32c_choose.cpp ${CMAKE_CURRENT_SOURCE_DIR}/port_srv/pg_crc32c_sse42.cpp) +endif() + SET(TGT_pgport_srv_INC ${PROJECT_SRC_DIR}/common/backend ${PROJECT_SRC_DIR}/common/port diff --git a/src/include/storage/lock/s_lock.h b/src/include/storage/lock/s_lock.h index dae744fa1..00e9c6663 100644 --- a/src/include/storage/lock/s_lock.h +++ b/src/include/storage/lock/s_lock.h @@ -349,7 +349,7 @@ static __inline__ int tas(volatile slock_t* lock) * later, so the compiler builtin is preferred if available. Note also that * the int-width variant of the builtin works on more chips than other widths. */ -#if defined(__arm__) || defined(__arm) +#if defined(__arm__) || defined(__arm) || defined(__riscv) #define HAS_TEST_AND_SET #define TAS(lock) tas(lock) @@ -365,7 +365,7 @@ static __inline__ int tas(volatile slock_t* lock) #define S_UNLOCK(lock) __sync_lock_release(lock) -#else /* !HAVE_GCC_INT_ATOMICS */ +#elif !defined(__riscv) /* !HAVE_GCC_INT_ATOMICS */ typedef unsigned char slock_t; diff --git a/src/lib/gstrace/CMakeLists.txt b/src/lib/gstrace/CMakeLists.txt index 4551e767c..803f369e9 100755 --- a/src/lib/gstrace/CMakeLists.txt +++ b/src/lib/gstrace/CMakeLists.txt @@ -1,6 +1,6 @@ #This is the main CMAKE for build all components. execute_process( - COMMAND python ./script/trc_gen.py -s ./config/ -o ${CMAKE_CURRENT_SOURCE_DIR}/../../include/gstrace + COMMAND python3 ./script/trc_gen.py -s ./config/ -o ${CMAKE_CURRENT_SOURCE_DIR}/../../include/gstrace WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) From 2d07706df5e9945bde1546964679ae0df009c15e Mon Sep 17 00:00:00 2001 From: nexplorer-3e <60149113+nexplorer-3e@users.noreply.github.com> Date: Thu, 15 Feb 2024 19:39:11 +0800 Subject: [PATCH 303/304] riscv: add makefile support TODO: enable mot & comment --- build/script/utils/common.sh | 2 +- config/config.guess | 3 +++ config/config.sub | 2 ++ .../storage/mot/core/infra/synchronization/cycles.h | 8 ++++++++ src/lib/gstrace/Makefile | 2 +- 5 files changed, 15 insertions(+), 2 deletions(-) diff --git a/build/script/utils/common.sh b/build/script/utils/common.sh index f85d99d6e..7ef42f1a8 100644 --- a/build/script/utils/common.sh +++ b/build/script/utils/common.sh @@ -108,7 +108,7 @@ if [ "$PLATFORM_ARCH"X == "loongarch64"X ];then gcc_version="8.3" fi -if [ "$PLATFORM_ARCH"X == "aarch64"X ] && [ "$gcc_version" == "10.3" ]; then +if ( [ "$PLATFORM_ARCH"X == "aarch64"X ] || [ "$PLATFORM_ARCH"X == "riscv64"X ] ) && [ "$gcc_version" == "10.3" ]; then gcc_sub_version="1" else gcc_sub_version="0" diff --git a/config/config.guess b/config/config.guess index 15f22ac76..09ffe4e94 100644 --- a/config/config.guess +++ b/config/config.guess @@ -960,6 +960,9 @@ EOF m68*:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-gnu exit ;; + riscv64:Linux:*:*) + echo "$UNAME_MACHINE"-linux-"$LIBC" + exit ;; loongarch*:Linux:*:*) echo "$UNAME_MACHINE"-linux-"$LIBC" exit ;; diff --git a/config/config.sub b/config/config.sub index 6e5c23c53..8439e7f6c 100644 --- a/config/config.sub +++ b/config/config.sub @@ -306,6 +306,7 @@ case $basic_machine in | powerpc | powerpc64 | powerpc64le | powerpcle \ | pyramid \ | rl78 | rx \ + | riscv64 \ | score \ | sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[34]eb | sheb | shbe | shle | sh[1234]le | sh3ele \ | sh64 | sh64le \ @@ -422,6 +423,7 @@ case $basic_machine in | powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* \ | pyramid-* \ | rl78-* | romp-* | rs6000-* | rx-* \ + | riscv64-* \ | sh-* | sh[1234]-* | sh[24]a-* | sh[24]aeb-* | sh[23]e-* | sh[34]eb-* | sheb-* | shbe-* \ | shle-* | sh[1234]le-* | sh3ele-* | sh64-* | sh64le-* \ | sparc-* | sparc64-* | sparc64b-* | sparc64v-* | sparc86x-* | sparclet-* \ diff --git a/src/gausskernel/storage/mot/core/infra/synchronization/cycles.h b/src/gausskernel/storage/mot/core/infra/synchronization/cycles.h index 89213af13..8791f1c28 100644 --- a/src/gausskernel/storage/mot/core/infra/synchronization/cycles.h +++ b/src/gausskernel/storage/mot/core/infra/synchronization/cycles.h @@ -55,6 +55,10 @@ class CpuCyclesLevelTime { unsigned long cval = 0; asm volatile("isb; mrs %0, cntvct_el0" : "=r"(cval) : : "memory"); return cval; +#elif defined(__riscv64) + unsigned long cval = 0; + asm volatile ("rdcycle %0" : "=r" (dst) ); + return cval; #else #error "Unsupported CPU architecture or compiler." #endif @@ -75,6 +79,10 @@ class CpuCyclesLevelTime { unsigned long cval = 0; asm volatile("isb; mrs %0, cntvct_el0" : "=r"(cval) : : "memory"); return cval; +#elif defined(__riscv64) + unsigned long cval = 0; + asm volatile ("rdcycle %0" : "=r" (dst) ); + return cval; #else #error "Unsupported CPU architecture or compiler." #endif diff --git a/src/lib/gstrace/Makefile b/src/lib/gstrace/Makefile index 048612ba2..959f4c4be 100644 --- a/src/lib/gstrace/Makefile +++ b/src/lib/gstrace/Makefile @@ -45,7 +45,7 @@ all: $(HEADERFILE) install: all $(HEADERFILE): $(CONFIGFILE) - python ./script/trc_gen.py -s ./config/ -o ./../../include/gstrace + python3 ./script/trc_gen.py -s ./config/ -o ./../../include/gstrace clean: rm -f ./../../include/gstrace/*_gstrace.h From a57a6953d7bf4a8868e89d922f8d9df0a4735c4f Mon Sep 17 00:00:00 2001 From: nexplorer-3e <60149113+nexplorer-3e@users.noreply.github.com> Date: Mon, 26 Feb 2024 10:43:15 +0800 Subject: [PATCH 304/304] [HACK] cmake: remove duplicate libgomp install install by path would cause file exist error --- src/CMakeLists.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2634d5281..ecc55194a 100755 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -275,8 +275,6 @@ if("${ENABLE_MOT}" STREQUAL "ON") endif() install(FILES ${GCC_LIB_PATH}/lib64/libgcc_s.so.1 DESTINATION lib) -install(DIRECTORY ${GCC_LIB_PATH}/lib64/ DESTINATION lib - FILES_MATCHING PATTERN "libgomp.so*") install(CODE "execute_process( COMMAND cp ${GCC_LIB_PATH}/lib64/libstdc++.so.6.0.${LIBSTD_SUB_VERSION} ${prefix_home}/lib/libstdc++.so.6