From e921b362f092ad317221eff7dc17335c5b8ab30f Mon Sep 17 00:00:00 2001 From: melkij Date: Tue, 18 May 2021 12:39:54 +0300 Subject: [PATCH 01/29] simple_prompt signature was changed --- bin/pgut/pgut.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/bin/pgut/pgut.c b/bin/pgut/pgut.c index 8804195f..a020a3e7 100644 --- a/bin/pgut/pgut.c +++ b/bin/pgut/pgut.c @@ -10,6 +10,10 @@ #include "postgres_fe.h" #include "libpq/pqsignal.h" +#if PG_VERSION_NUM >= 140000 +#include "common/string.h" /* for simple_prompt */ +#endif + #include #include #include @@ -450,7 +454,7 @@ prompt_for_password(void) passwdbuf = pgut_malloc(BUFSIZE); memcpy(passwdbuf, buf, sizeof(char)*BUFSIZE); } -#else +#elif PG_VERSION_NUM < 140000 buf = pgut_malloc(BUFSIZE); if (have_passwd) { memcpy(buf, passwdbuf, sizeof(char)*BUFSIZE); @@ -461,6 +465,16 @@ prompt_for_password(void) passwdbuf = pgut_malloc(BUFSIZE); memcpy(passwdbuf, buf, sizeof(char)*BUFSIZE); } +#else + if (have_passwd) { + buf = pgut_malloc(BUFSIZE); + memcpy(buf, passwdbuf, sizeof(char)*BUFSIZE); + } else { + buf = simple_prompt("Password: ", false); + have_passwd = true; + passwdbuf = pgut_malloc(BUFSIZE); + memcpy(passwdbuf, buf, sizeof(char)*BUFSIZE); + } #endif if (buf == NULL) From 6f615d501038f257b96bac33b0e21f2d7fc1f3d1 Mon Sep 17 00:00:00 2001 From: melkij Date: Tue, 18 May 2021 12:46:40 +0300 Subject: [PATCH 02/29] Use SearchSysCacheCopy1 macro instead of SearchSysCacheCopy direct call as more future-proof Per syscache.h: > The use of the macros below rather than direct calls to the corresponding > functions is encouraged, as it insulates the caller from changes in the > maximum number of keys. Also this fixes segfault on pg14, but still not sure why exactly. --- lib/repack.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/repack.c b/lib/repack.c index 2690a9ce..96660c0d 100644 --- a/lib/repack.c +++ b/lib/repack.c @@ -1167,16 +1167,12 @@ swap_heap_or_index_files(Oid r1, Oid r2) relRelation = heap_open(RelationRelationId, RowExclusiveLock); #endif - reltup1 = SearchSysCacheCopy(RELOID, - ObjectIdGetDatum(r1), - 0, 0, 0); + reltup1 = SearchSysCacheCopy1(RELOID, ObjectIdGetDatum(r1)); if (!HeapTupleIsValid(reltup1)) elog(ERROR, "cache lookup failed for relation %u", r1); relform1 = (Form_pg_class) GETSTRUCT(reltup1); - reltup2 = SearchSysCacheCopy(RELOID, - ObjectIdGetDatum(r2), - 0, 0, 0); + reltup2 = SearchSysCacheCopy1(RELOID, ObjectIdGetDatum(r2)); if (!HeapTupleIsValid(reltup2)) elog(ERROR, "cache lookup failed for relation %u", r2); relform2 = (Form_pg_class) GETSTRUCT(reltup2); From 66ce5246af2e49a4cc09d609f1917e377d97982f Mon Sep 17 00:00:00 2001 From: melkij Date: Tue, 18 May 2021 13:19:14 +0300 Subject: [PATCH 03/29] Get rid the custom array_accum aggregate in favor of postgresql's built-in array_agg and string_agg. In postgresql 14, the signature of array_append was changed, and our create aggregate throws the error "function array_append(anyarray, anyelement) does not exist". Postgresql's built-in string_agg was introduced in postgresql 9.0, array_agg was from 8.4 release. Both are too old and no longer supported by pg_repack. So, instead of fixing the repack.array_accum, I want to drop it. One notable behavior difference is handling empty sets: array_agg will produce NULL. So I put several coalesce to avoid altering the query results. --- bin/pg_repack.c | 6 +++--- lib/pg_repack.sql.in | 31 ++++++++++++------------------- 2 files changed, 15 insertions(+), 22 deletions(-) diff --git a/bin/pg_repack.c b/bin/pg_repack.c index 38d40e22..ff39a84f 100644 --- a/bin/pg_repack.c +++ b/bin/pg_repack.c @@ -75,7 +75,7 @@ const char *PROGRAM_VERSION = "unknown"; * pg_regress. */ #define SQL_XID_SNAPSHOT_90200 \ - "SELECT repack.array_accum(l.virtualtransaction) " \ + "SELECT coalesce(array_agg(l.virtualtransaction), '{}') " \ " FROM pg_locks AS l " \ " LEFT JOIN pg_stat_activity AS a " \ " ON l.pid = a.pid " \ @@ -90,7 +90,7 @@ const char *PROGRAM_VERSION = "unknown"; " AND ((d.datname IS NULL OR d.datname = current_database()) OR l.database = 0)" #define SQL_XID_SNAPSHOT_90000 \ - "SELECT repack.array_accum(l.virtualtransaction) " \ + "SELECT coalesce(array_agg(l.virtualtransaction), '{}') " \ " FROM pg_locks AS l " \ " LEFT JOIN pg_stat_activity AS a " \ " ON l.pid = a.procpid " \ @@ -108,7 +108,7 @@ const char *PROGRAM_VERSION = "unknown"; * the WHERE clause is just to eat the $2 parameter (application name). */ #define SQL_XID_SNAPSHOT_80300 \ - "SELECT repack.array_accum(l.virtualtransaction) " \ + "SELECT coalesce(array_agg(l.virtualtransaction), '{}') " \ " FROM pg_locks AS l" \ " LEFT JOIN pg_stat_activity AS a " \ " ON l.pid = a.procpid " \ diff --git a/lib/pg_repack.sql.in b/lib/pg_repack.sql.in index ebbf9f2c..23305dba 100644 --- a/lib/pg_repack.sql.in +++ b/lib/pg_repack.sql.in @@ -16,13 +16,6 @@ CREATE FUNCTION repack.version_sql() RETURNS text AS $$SELECT 'pg_repack REPACK_VERSION'::text$$ LANGUAGE SQL IMMUTABLE STRICT; -CREATE AGGREGATE repack.array_accum ( - sfunc = array_append, - basetype = anyelement, - stype = anyarray, - initcond = '{}' -); - -- Always specify search_path to 'pg_catalog' so that we -- always can get schema-qualified relation name CREATE FUNCTION repack.oid2text(oid) RETURNS text AS @@ -33,7 +26,7 @@ LANGUAGE sql STABLE STRICT SET search_path to 'pg_catalog'; CREATE FUNCTION repack.get_index_columns(oid, text) RETURNS text AS $$ - SELECT array_to_string(repack.array_accum(quote_ident(attname)), $2) + SELECT coalesce(string_agg(quote_ident(attname), $2), '') FROM pg_attribute, (SELECT indrelid, indkey, @@ -53,8 +46,8 @@ LANGUAGE C STABLE STRICT; CREATE FUNCTION repack.get_create_index_type(oid, name) RETURNS text AS $$ SELECT 'CREATE TYPE ' || $2 || ' AS (' || - array_to_string(repack.array_accum(quote_ident(attname) || ' ' || - pg_catalog.format_type(atttypid, atttypmod)), ', ') || ')' + coalesce(string_agg(quote_ident(attname) || ' ' || + pg_catalog.format_type(atttypid, atttypmod), ', '), '') || ')' FROM pg_attribute, (SELECT indrelid, indkey, @@ -90,9 +83,9 @@ LANGUAGE sql STABLE STRICT; CREATE FUNCTION repack.get_assign(oid, text) RETURNS text AS $$ - SELECT '(' || array_to_string(repack.array_accum(quote_ident(attname)), ', ') || + SELECT '(' || coalesce(string_agg(quote_ident(attname), ', '), '') || ') = (' || $2 || '.' || - array_to_string(repack.array_accum(quote_ident(attname)), ', ' || $2 || '.') || ')' + coalesce(string_agg(quote_ident(attname), ', ' || $2 || '.'), '') || ')' FROM (SELECT attname FROM pg_attribute WHERE attrelid = $1 AND attnum > 0 AND NOT attisdropped ORDER BY attnum) tmp; @@ -102,9 +95,9 @@ LANGUAGE sql STABLE STRICT; CREATE FUNCTION repack.get_compare_pkey(oid, text) RETURNS text AS $$ - SELECT '(' || array_to_string(repack.array_accum(quote_ident(attname)), ', ') || + SELECT '(' || coalesce(string_agg(quote_ident(attname), ', '), '') || ') = (' || $2 || '.' || - array_to_string(repack.array_accum(quote_ident(attname)), ', ' || $2 || '.') || ')' + coalesce(string_agg(quote_ident(attname), ', ' || $2 || '.'), '') || ')' FROM pg_attribute, (SELECT indrelid, indkey, @@ -122,7 +115,7 @@ LANGUAGE sql STABLE STRICT; CREATE FUNCTION repack.get_columns_for_create_as(oid) RETURNS text AS $$ -SELECT array_to_string(repack.array_accum(c), ',') FROM (SELECT +SELECT coalesce(string_agg(c, ','), '') FROM (SELECT CASE WHEN attisdropped THEN 'NULL::integer AS ' || quote_ident(attname) ELSE quote_ident(attname) @@ -142,7 +135,7 @@ SELECT 'ALTER TABLE ' || $2 || ' ' || array_to_string(dropped_columns, ', ') FROM ( SELECT - repack.array_accum('DROP COLUMN ' || quote_ident(attname)) AS dropped_columns + array_agg('DROP COLUMN ' || quote_ident(attname)) AS dropped_columns FROM ( SELECT * FROM pg_attribute WHERE attrelid = $1 AND attnum > 0 AND attisdropped @@ -161,7 +154,7 @@ LANGUAGE sql STABLE STRICT; CREATE FUNCTION repack.get_storage_param(oid) RETURNS TEXT AS $$ -SELECT array_to_string(array_agg(param), ', ') +SELECT string_agg(param, ', ') FROM ( -- table storage parameter SELECT unnest(reloptions) as param @@ -196,7 +189,7 @@ $$ SELECT 'ALTER TABLE repack.table_' || $1 || array_to_string(column_storage, ',') FROM ( SELECT - repack.array_accum(' ALTER ' || quote_ident(attname) || + array_agg(' ALTER ' || quote_ident(attname) || CASE attstorage WHEN 'p' THEN ' SET STORAGE PLAIN' WHEN 'm' THEN ' SET STORAGE MAIN' @@ -222,7 +215,7 @@ LANGUAGE sql STABLE STRICT; -- includes not only PRIMARY KEYS but also UNIQUE NOT NULL keys CREATE VIEW repack.primary_keys AS - SELECT indrelid, (repack.array_accum(indexrelid))[1] AS indexrelid + SELECT indrelid, min(indexrelid) AS indexrelid FROM (SELECT indrelid, indexrelid FROM pg_index WHERE indisunique AND indisvalid From 74621d710d0f9651638f5c7b8ead43972517fb60 Mon Sep 17 00:00:00 2001 From: melkij Date: Mon, 7 Jun 2021 15:35:51 +0300 Subject: [PATCH 04/29] Check for the existence of the tables specified by --table or --parent-table PostgreSQL 14 will produce something like ERROR: pg_repack failed with error: ERROR: relation "dummy_table" does not exist CONTEXT: unnamed portal parameter $2 = '...' The second line looks weird and breaks tests. The second word "ERROR" also looks strange. So an explicit check for the existence of a table has been added. --- bin/pg_repack.c | 110 ++++++++++++++++++++++++++++++ regress/expected/repack-check.out | 2 +- 2 files changed, 111 insertions(+), 1 deletion(-) diff --git a/bin/pg_repack.c b/bin/pg_repack.c index ff39a84f..db2c865d 100644 --- a/bin/pg_repack.c +++ b/bin/pg_repack.c @@ -212,6 +212,7 @@ typedef struct repack_table static bool is_superuser(void); static void check_tablespace(void); static bool preliminary_checks(char *errbuf, size_t errsize); +static bool is_requested_relation_exists(char *errbuf, size_t errsize); static void repack_all_databases(const char *order_by); static bool repack_one_database(const char *order_by, char *errbuf, size_t errsize); static void repack_one_table(repack_table *table, const char *order_by); @@ -558,6 +559,109 @@ preliminary_checks(char *errbuf, size_t errsize){ return ret; } +/* + * Check the presence of tables specified by --parent-table and --table + * otherwise format user-friendly message + */ +static bool +is_requested_relation_exists(char *errbuf, size_t errsize){ + bool ret = false; + PGresult *res = NULL; + const char **params = NULL; + int iparam = 0; + StringInfoData sql; + int num_relations; + SimpleStringListCell *cell; + + num_relations = simple_string_list_size(parent_table_list) + + simple_string_list_size(table_list); + + /* nothing was implicitly requested, so nothing to do here */ + if (num_relations == 0) + return true; + + params = pgut_malloc(num_relations * sizeof(char *)); + initStringInfo(&sql); + appendStringInfoString(&sql, "SELECT r FROM (VALUES "); + + for (cell = table_list.head; cell; cell = cell->next) + { + appendStringInfo(&sql, "($%d)", iparam + 1); + params[iparam++] = cell->val; + if (iparam < num_relations) + appendStringInfoChar(&sql, ','); + } + for (cell = parent_table_list.head; cell; cell = cell->next) + { + appendStringInfo(&sql, "($%d)", iparam + 1); + params[iparam++] = cell->val; + if (iparam < num_relations) + appendStringInfoChar(&sql, ','); + } + appendStringInfoString(&sql, + ") AS given_t(r)" + " WHERE NOT EXISTS(" + " SELECT FROM repack.tables WHERE relid=to_regclass(given_t.r) )" + ); + + /* double check the parameters array is sane */ + if (iparam != num_relations) + { + if (errbuf) + snprintf(errbuf, errsize, + "internal error: bad parameters count: %i instead of %i", + iparam, num_relations); + goto cleanup; + } + + res = execute_elevel(sql.data, iparam, params, DEBUG2); + if (PQresultStatus(res) == PGRES_TUPLES_OK) + { + int num; + + num = PQntuples(res); + + if (num != 0) + { + int i; + StringInfoData rel_names; + initStringInfo(&rel_names); + + for (i = 0; i < num; i++) + { + appendStringInfo(&rel_names, "\"%s\"", getstr(res, i, 0)); + if ((i + 1) != num) + appendStringInfoString(&rel_names, ", "); + } + + if (errbuf) + { + if (num > 1) + snprintf(errbuf, errsize, + "relations do not exist: %s", rel_names.data); + else + snprintf(errbuf, errsize, + "relation %s does not exist", rel_names.data); + } + termStringInfo(&rel_names); + } + else + ret = true; + } + else + { + if (errbuf) + snprintf(errbuf, errsize, "%s", PQerrorMessage(connection)); + } + CLEARPGRES(res); + +cleanup: + CLEARPGRES(res); + termStringInfo(&sql); + free(params); + return ret; +} + /* * Call repack_one_database for each database. */ @@ -657,6 +761,9 @@ repack_one_database(const char *orderby, char *errbuf, size_t errsize) if (!preliminary_checks(errbuf, errsize)) goto cleanup; + if (!is_requested_relation_exists(errbuf, errsize)) + goto cleanup; + /* acquire target tables */ appendStringInfoString(&sql, "SELECT t.*," @@ -2115,6 +2222,9 @@ repack_all_indexes(char *errbuf, size_t errsize) if (!preliminary_checks(errbuf, errsize)) goto cleanup; + if (!is_requested_relation_exists(errbuf, errsize)) + goto cleanup; + if (r_index.head) { appendStringInfoString(&sql, diff --git a/regress/expected/repack-check.out b/regress/expected/repack-check.out index 0da1133e..bc586313 100644 --- a/regress/expected/repack-check.out +++ b/regress/expected/repack-check.out @@ -285,7 +285,7 @@ CREATE TABLE child_b_1(val integer primary key) INHERITS(parent_b); CREATE TABLE child_b_2(val integer primary key) INHERITS(parent_b); -- => ERROR \! pg_repack --dbname=contrib_regression --parent-table=dummy_table -ERROR: pg_repack failed with error: ERROR: relation "dummy_table" does not exist +ERROR: pg_repack failed with error: relation "dummy_table" does not exist -- => ERROR \! pg_repack --dbname=contrib_regression --parent-table=dummy_index --index=dummy_index ERROR: cannot specify --index (-i) and --parent-table (-I) From d07030edf46b1367675bd2f40e6db1fc1abca3a2 Mon Sep 17 00:00:00 2001 From: melkij Date: Mon, 7 Jun 2021 16:07:02 +0300 Subject: [PATCH 05/29] Prepare release 1.4.7 First sketch to see what buildfarm thinks. Also recheck PGVER=9.6 build --- .travis.yml | 4 +++- META.json | 4 ++-- doc/pg_repack.rst | 6 +++++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index fcd27df8..327c5eb5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,13 +4,15 @@ dist: xenial sudo: required env: + - PGVER=14 + PGTESTING=1 - PGVER=13 - PGVER=12 - PGVER=11 - PGVER=10 # Disabled because packages broken at least on xenial # https://www.postgresql.org/message-id/CA%2Bmi_8a1oEnCzkt0CvqysgY4MQ6jEefjmS%3Dq_K-AvOx%3DF7m2%2BQ%40mail.gmail.com - # - PGVER=9.6 + - PGVER=9.6 - PGVER=9.5 - PGVER=9.4 diff --git a/META.json b/META.json index d12358b1..40341cb5 100644 --- a/META.json +++ b/META.json @@ -2,7 +2,7 @@ "name": "pg_repack", "abstract": "PostgreSQL module for data reorganization", "description": "Reorganize tables in PostgreSQL databases with minimal locks", - "version": "1.4.6", + "version": "1.4.7", "maintainer": [ "Beena Emerson ", "Josh Kupershmidt ", @@ -15,7 +15,7 @@ "provides": { "pg_repack": { "file": "lib/pg_repack.sql", - "version": "1.4.6", + "version": "1.4.7", "abstract": "Reorganize tables in PostgreSQL databases with minimal locks" } }, diff --git a/doc/pg_repack.rst b/doc/pg_repack.rst index 7d2ff174..90834b92 100644 --- a/doc/pg_repack.rst +++ b/doc/pg_repack.rst @@ -40,7 +40,7 @@ Requirements ------------ PostgreSQL versions - PostgreSQL 9.4, 9.5, 9.6, 10, 11, 12, 13 + PostgreSQL 9.4, 9.5, 9.6, 10, 11, 12, 13, 14 Disks Performing a full-table repack requires free disk space about twice as @@ -466,6 +466,10 @@ Creating indexes concurrently comes with a few caveats, please see `the document Releases -------- +* pg_repack 1.4.7 + + * Added support for PostgreSQL 14 + * pg_repack 1.4.6 * Added support for PostgreSQL 13 From f9422a6f477c577a5e1f41bc869da24b92ba949b Mon Sep 17 00:00:00 2001 From: melkij Date: Mon, 7 Jun 2021 17:31:04 +0300 Subject: [PATCH 06/29] ubuntu xenial is EOL, try focal 20.04 LTS --- .travis.yml | 2 +- regress/travis_prepare.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 327c5eb5..fd0a66a1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ # Travis CI configuration file for psycopg2 -dist: xenial +dist: focal sudo: required env: diff --git a/regress/travis_prepare.sh b/regress/travis_prepare.sh index 8e105a4a..2c1a2da8 100755 --- a/regress/travis_prepare.sh +++ b/regress/travis_prepare.sh @@ -18,7 +18,7 @@ sudo sed -i "s/main[[:space:]]*$/main ${PGVER}/" \ /etc/apt/sources.list.d/pgdg.list if [ "$PGTESTING" != "" ]; then - sudo sed -i "s/xenial-pgdg/xenial-pgdg-testing/" \ + sudo sed -i "s/focal-pgdg/focal-pgdg-testing/" \ /etc/apt/sources.list.d/pgdg.list fi From 68453c54a0489c074cf660d6e8fe8e7ecd74ad14 Mon Sep 17 00:00:00 2001 From: melkij Date: Mon, 7 Jun 2021 17:44:19 +0300 Subject: [PATCH 07/29] third party apt-repositories removed from the Bionic build image. Next try https://docs.travis-ci.com/user/reference/bionic/#third-party-apt-repositories-removed --- regress/travis_prepare.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/regress/travis_prepare.sh b/regress/travis_prepare.sh index 2c1a2da8..74a4020d 100755 --- a/regress/travis_prepare.sh +++ b/regress/travis_prepare.sh @@ -14,8 +14,10 @@ sudo apt-get remove -y libpq5 # Match libpq and server-dev packages # See https://github.com/reorg/pg_repack/issues/63 -sudo sed -i "s/main[[:space:]]*$/main ${PGVER}/" \ - /etc/apt/sources.list.d/pgdg.list +sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg ${PGVER}" > /etc/apt/sources.list.d/pgdg.list' + +# Import the repository signing key: +wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - if [ "$PGTESTING" != "" ]; then sudo sed -i "s/focal-pgdg/focal-pgdg-testing/" \ From 4986f69b8f145a03716931ae26fbffcb174c4ba5 Mon Sep 17 00:00:00 2001 From: melkij Date: Mon, 7 Jun 2021 17:55:51 +0300 Subject: [PATCH 08/29] amd64 packages only, please --- regress/travis_prepare.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regress/travis_prepare.sh b/regress/travis_prepare.sh index 74a4020d..8c59b0bb 100755 --- a/regress/travis_prepare.sh +++ b/regress/travis_prepare.sh @@ -14,7 +14,7 @@ sudo apt-get remove -y libpq5 # Match libpq and server-dev packages # See https://github.com/reorg/pg_repack/issues/63 -sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg ${PGVER}" > /etc/apt/sources.list.d/pgdg.list' +sudo sh -c 'echo "deb [arch=amd64] http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg ${PGVER}" > /etc/apt/sources.list.d/pgdg.list' # Import the repository signing key: wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - From 51723d63d4f5c120d6bb7c11e3ed7981309b7785 Mon Sep 17 00:00:00 2001 From: melkij Date: Mon, 7 Jun 2021 18:10:22 +0300 Subject: [PATCH 09/29] apt repository section was changed? --- regress/travis_prepare.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regress/travis_prepare.sh b/regress/travis_prepare.sh index 8c59b0bb..ab46d181 100755 --- a/regress/travis_prepare.sh +++ b/regress/travis_prepare.sh @@ -14,7 +14,7 @@ sudo apt-get remove -y libpq5 # Match libpq and server-dev packages # See https://github.com/reorg/pg_repack/issues/63 -sudo sh -c 'echo "deb [arch=amd64] http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg ${PGVER}" > /etc/apt/sources.list.d/pgdg.list' +sudo sh -c 'echo "deb [arch=amd64] http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main ${PGVER}" > /etc/apt/sources.list.d/pgdg.list' # Import the repository signing key: wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - From ef050ac59e0569e9e73f1f2f5b361c631c2318f7 Mon Sep 17 00:00:00 2001 From: melkij Date: Mon, 7 Jun 2021 18:18:02 +0300 Subject: [PATCH 10/29] postgresql-14 beta1 should be already in repo, retry --- .travis.yml | 1 - regress/travis_prepare.sh | 5 ----- 2 files changed, 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index fd0a66a1..06b38614 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,6 @@ sudo: required env: - PGVER=14 - PGTESTING=1 - PGVER=13 - PGVER=12 - PGVER=11 diff --git a/regress/travis_prepare.sh b/regress/travis_prepare.sh index ab46d181..7d045c02 100755 --- a/regress/travis_prepare.sh +++ b/regress/travis_prepare.sh @@ -19,11 +19,6 @@ sudo sh -c 'echo "deb [arch=amd64] http://apt.postgresql.org/pub/repos/apt $(lsb # Import the repository signing key: wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - -if [ "$PGTESTING" != "" ]; then - sudo sed -i "s/focal-pgdg/focal-pgdg-testing/" \ - /etc/apt/sources.list.d/pgdg.list -fi - sudo apt-get update # This might be a moving target, but it currently fails. 13 could start From c693ea0eba044566a079da65222bab5105cbdd6a Mon Sep 17 00:00:00 2001 From: melkij Date: Mon, 7 Jun 2021 18:22:48 +0300 Subject: [PATCH 11/29] try a workaround for pg9.5, 9.6 Possible we decide to drop these versions, but just check --- bin/pg_repack.c | 6 +++++- regress/expected/repack-check.out | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/bin/pg_repack.c b/bin/pg_repack.c index db2c865d..1bdceb51 100644 --- a/bin/pg_repack.c +++ b/bin/pg_repack.c @@ -580,6 +580,10 @@ is_requested_relation_exists(char *errbuf, size_t errsize){ if (num_relations == 0) return true; + /* has no suitable to_regclass(text) */ + if (PQserverVersion(connection)<90600) + return true; + params = pgut_malloc(num_relations * sizeof(char *)); initStringInfo(&sql); appendStringInfoString(&sql, "SELECT r FROM (VALUES "); @@ -641,7 +645,7 @@ is_requested_relation_exists(char *errbuf, size_t errsize){ "relations do not exist: %s", rel_names.data); else snprintf(errbuf, errsize, - "relation %s does not exist", rel_names.data); + "ERROR: relation %s does not exist", rel_names.data); } termStringInfo(&rel_names); } diff --git a/regress/expected/repack-check.out b/regress/expected/repack-check.out index bc586313..0da1133e 100644 --- a/regress/expected/repack-check.out +++ b/regress/expected/repack-check.out @@ -285,7 +285,7 @@ CREATE TABLE child_b_1(val integer primary key) INHERITS(parent_b); CREATE TABLE child_b_2(val integer primary key) INHERITS(parent_b); -- => ERROR \! pg_repack --dbname=contrib_regression --parent-table=dummy_table -ERROR: pg_repack failed with error: relation "dummy_table" does not exist +ERROR: pg_repack failed with error: ERROR: relation "dummy_table" does not exist -- => ERROR \! pg_repack --dbname=contrib_regression --parent-table=dummy_index --index=dummy_index ERROR: cannot specify --index (-i) and --parent-table (-I) From d263fc3aa337194493168054711be22d231f8948 Mon Sep 17 00:00:00 2001 From: melkij Date: Mon, 7 Jun 2021 18:31:50 +0300 Subject: [PATCH 12/29] missing build dependency by postgresql-server-dev lz4 --- regress/travis_prepare.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/regress/travis_prepare.sh b/regress/travis_prepare.sh index 7d045c02..cee73cc5 100755 --- a/regress/travis_prepare.sh +++ b/regress/travis_prepare.sh @@ -29,6 +29,11 @@ if [[ "$PGVER" = "9.4" ]]; then sudo apt-mark hold libpq5 fi +# missing build dependency by postgresql-server-dev +if [[ "$PGVER" -ge "14" ]]; then + sudo apt-get install -y liblz4-dev +fi + if ! sudo apt-get install -y \ postgresql-$PGVER \ postgresql-client-$PGVER \ From af02f3b4240404ce584d2df2c95fba5414595319 Mon Sep 17 00:00:00 2001 From: Valeriya Popova Date: Wed, 7 Jul 2021 16:29:58 +0300 Subject: [PATCH 13/29] update docs broken links --- META.json | 4 ++-- README.rst | 4 ++-- doc/pg_repack.rst | 3 +-- doc/pg_repack_jp.rst | 3 +-- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/META.json b/META.json index d12358b1..1dacb3df 100644 --- a/META.json +++ b/META.json @@ -27,7 +27,7 @@ } }, "resources": { - "homepage": "http://reorg.github.com/pg_repack", + "homepage": "https://reorg.github.io/pg_repack", "bugtracker": { "web": "https://github.com/reorg/pg_repack/issues" }, @@ -39,6 +39,6 @@ }, "meta-spec": { "version": "1.0.0", - "url": "http://pgxn.org/meta/spec.txt" + "url": "https://pgxn.org/meta/spec.txt" } } diff --git a/README.rst b/README.rst index ef10b317..247be93d 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,7 @@ pg_repack -- Reorganize tables in PostgreSQL databases with minimal locks ========================================================================= -- Homepage: https://reorg.github.com/pg_repack +- Homepage: https://reorg.github.io/pg_repack - Download: https://pgxn.org/dist/pg_repack/ - Development: https://github.com/reorg/pg_repack - Bug Report: https://github.com/reorg/pg_repack/issues @@ -22,7 +22,7 @@ CLUSTER directly. Please check the documentation (in the ``doc`` directory or online_) for installation and usage instructions. -.. _pg_repack: https://reorg.github.com/pg_repack +.. _pg_repack: https://reorg.github.io/pg_repack .. _CLUSTER: https://www.postgresql.org/docs/current/static/sql-cluster.html .. _VACUUM FULL: VACUUM_ .. _VACUUM: https://www.postgresql.org/docs/current/static/sql-vacuum.html diff --git a/doc/pg_repack.rst b/doc/pg_repack.rst index 7d2ff174..c89627f4 100644 --- a/doc/pg_repack.rst +++ b/doc/pg_repack.rst @@ -28,7 +28,7 @@ NOTICE: * Target table must have a PRIMARY KEY, or at least a UNIQUE total index on a NOT NULL column. -.. _pg_repack: http://reorg.github.com/pg_repack +.. _pg_repack: https://reorg.github.io/pg_repack .. _CLUSTER: http://www.postgresql.org/docs/current/static/sql-cluster.html .. _VACUUM FULL: VACUUM_ .. _VACUUM: http://www.postgresql.org/docs/current/static/sql-vacuum.html @@ -587,4 +587,3 @@ See Also * `clusterdb `__ * `vacuumdb `__ - diff --git a/doc/pg_repack_jp.rst b/doc/pg_repack_jp.rst index 8f314992..b4ac5c5c 100644 --- a/doc/pg_repack_jp.rst +++ b/doc/pg_repack_jp.rst @@ -50,7 +50,7 @@ pg_repackでは再編成する方法として次のものが選択できます * DBのスーパーユーザだけがpg_repackを実行できます * 対象となるテーブルは主キー、もしくはNOT NULL制約を持つカラムへのユニーク制約をもつインデックスが存在している必要があります -.. _pg_repack: http://reorg.github.com/pg_repack +.. _pg_repack: https://reorg.github.io/pg_repack .. _CLUSTER: http://www.postgresql.jp/document/current/html/sql-cluster.html .. _VACUUM FULL: VACUUM_ .. _VACUUM: http://www.postgresql.jp/document/current/html/sql-vacuum.html @@ -1050,4 +1050,3 @@ ACCESS EXCLUSIVEロックを取得します。その他のステップでは、A * `clusterdb `__ * `vacuumdb `__ - From 76a38e2960955214f233b61da295a49acebeda81 Mon Sep 17 00:00:00 2001 From: Daniel Merken Date: Wed, 25 Aug 2021 17:19:11 -0700 Subject: [PATCH 14/29] Remove connection info from error log --- bin/pgut/pgut.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/pgut/pgut.c b/bin/pgut/pgut.c index 8804195f..64cf13d2 100644 --- a/bin/pgut/pgut.c +++ b/bin/pgut/pgut.c @@ -548,8 +548,7 @@ pgut_connect(const char *info, YesNo prompt, int elevel) ereport(elevel, (errcode(E_PG_CONNECT), - errmsg("could not connect to database with \"%s\": %s", - info, PQerrorMessage(conn)))); + errmsg("could not connect to database: %s", PQerrorMessage(conn)))); PQfinish(conn); return NULL; } From 37d3dba732752ed7b2552f9770a19ffa5af59c52 Mon Sep 17 00:00:00 2001 From: lincuiping <57204139+lincuiping@users.noreply.github.com> Date: Tue, 16 Nov 2021 16:27:58 +0800 Subject: [PATCH 15/29] remove unnecessary plus operator --- bin/pgut/pgut.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/pgut/pgut.c b/bin/pgut/pgut.c index 7ae1ad02..8ece311b 100644 --- a/bin/pgut/pgut.c +++ b/bin/pgut/pgut.c @@ -346,7 +346,7 @@ parse_time(const char *value, time_t *time) char junk[2]; /* tmp = replace( value, !isalnum, ' ' ) */ - tmp = pgut_malloc(strlen(value) + + 1); + tmp = pgut_malloc(strlen(value) + 1); len = 0; for (i = 0; value[i]; i++) tmp[len++] = (IsAlnum(value[i]) ? value[i] : ' '); From 81f26d10e5e5b0c3fb7afbd5d91f3a6f3dfafd87 Mon Sep 17 00:00:00 2001 From: sunhm89 <93200340+sunhm89@users.noreply.github.com> Date: Mon, 22 Nov 2021 15:52:05 +0800 Subject: [PATCH 16/29] fix typo in pgut.c --- bin/pgut/pgut.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/pgut/pgut.c b/bin/pgut/pgut.c index 8ece311b..2cd796d1 100644 --- a/bin/pgut/pgut.c +++ b/bin/pgut/pgut.c @@ -1164,7 +1164,7 @@ on_interrupt(void) pgutConn *c; int save_errno = errno; - /* Set interruped flag */ + /* Set interrupted flag */ interrupted = true; if (in_cleanup) From ddd5f1439c3e0221fb0c4a251aaf08687e08c3ed Mon Sep 17 00:00:00 2001 From: zhuqx-fnst <94515051+zhuqx-fnst@users.noreply.github.com> Date: Thu, 2 Dec 2021 15:18:22 +0800 Subject: [PATCH 17/29] Fix typo Fix typo --- lib/pg_repack.sql.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pg_repack.sql.in b/lib/pg_repack.sql.in index 23305dba..ea45b4f7 100644 --- a/lib/pg_repack.sql.in +++ b/lib/pg_repack.sql.in @@ -147,8 +147,8 @@ WHERE $$ LANGUAGE sql STABLE STRICT; --- Get a comma-separated storage paramter for the table including --- paramters for the corresponding TOAST table. +-- Get a comma-separated storage parameter for the table including +-- parameters for the corresponding TOAST table. -- Note that since oid setting is always not NULL, this function -- never returns NULL CREATE FUNCTION repack.get_storage_param(oid) From 4211335b59617cf280fea2fa4b7326b89a88dbb2 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 6 Dec 2021 22:42:19 +0100 Subject: [PATCH 18/29] Reassure the user that it's ok to drop the extension See #281 --- bin/pg_repack.c | 3 ++- doc/pg_repack.rst | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/pg_repack.c b/bin/pg_repack.c index 1bdceb51..940a0d81 100644 --- a/bin/pg_repack.c +++ b/bin/pg_repack.c @@ -515,7 +515,8 @@ preliminary_checks(char *errbuf, size_t errsize){ { if (errbuf) snprintf(errbuf, errsize, - "extension '%s' required, found extension '%s'", + "extension '%s' required, found '%s';" + " please drop and re-create the extension", buf, libver); goto cleanup; } diff --git a/doc/pg_repack.rst b/doc/pg_repack.rst index b79f0bd8..a5efc22d 100644 --- a/doc/pg_repack.rst +++ b/doc/pg_repack.rst @@ -356,7 +356,7 @@ ERROR: program 'pg_repack V1' does not match database library 'pg_repack V2' database; if they are what expected you may need to repeat pg_repack installation. -ERROR: extension 'pg_repack V1' required, found extension 'pg_repack V2' +ERROR: extension 'pg_repack V1' required, found 'pg_repack V2' The SQL extension found in the database does not match the version required by the pg_repack program. From 944dbf7a49adfed7bd2ad467ae9431e8067442a4 Mon Sep 17 00:00:00 2001 From: melkij Date: Thu, 19 May 2022 22:28:23 +0300 Subject: [PATCH 19/29] prepare support for postgresql 15 minor change, postgresql commit 98e93a1fc93e9b54eb477d870ec744e9e1669f34 removed include of pwd.h and breaks build. I decided to replace your own implementation of get_username and use the generic get_user_name_or_exit from the postgresql code. Exists since postgresql 9.4, so no change in supported versions. --- bin/pgut/pgut-fe.c | 40 +++------------------------------------- bin/pgut/pgut-fe.h | 2 +- 2 files changed, 4 insertions(+), 38 deletions(-) diff --git a/bin/pgut/pgut-fe.c b/bin/pgut/pgut-fe.c index 56262fa9..8633ddac 100644 --- a/bin/pgut/pgut-fe.c +++ b/bin/pgut/pgut-fe.c @@ -9,6 +9,7 @@ #define FRONTEND #include "pgut-fe.h" +#include "common/username.h" #ifdef HAVE_GETOPT_H #include @@ -16,7 +17,7 @@ #include #endif -char *dbname = NULL; +const char *dbname = NULL; char *host = NULL; char *port = NULL; char *username = NULL; @@ -33,8 +34,6 @@ worker_conns workers = { static bool parse_pair(const char buffer[], char key[], char value[]); -static char *get_username(void); - /* * Set up worker conns which will be used for concurrent index rebuilds. @@ -638,39 +637,6 @@ option_find(int c, pgut_option opts1[], pgut_option opts2[]) return NULL; /* not found */ } -/* - * Returns the current user name. - */ -static char * -get_username(void) -{ - char *ret; - -#ifndef WIN32 - struct passwd *pw; - - pw = getpwuid(geteuid()); - ret = (pw ? pw->pw_name : NULL); -#else - static char username[128]; /* remains after function execute */ - DWORD len = sizeof(username) - 1; - - if (GetUserNameA(username, &len)) - ret = username; - else - { - _dosmaperr(GetLastError()); - ret = NULL; - } -#endif - - if (ret == NULL) - ereport(ERROR, - (errcode_errno(), - errmsg("could not get current user name: "))); - return ret; -} - static int option_has_arg(char type) { @@ -787,7 +753,7 @@ pgut_getopt(int argc, char **argv, pgut_option options[]) (void) (dbname || (dbname = getenv("PGDATABASE")) || (dbname = getenv("PGUSER")) || - (dbname = get_username())); + (dbname = get_user_name_or_exit(PROGRAM_NAME))); return optind; } diff --git a/bin/pgut/pgut-fe.h b/bin/pgut/pgut-fe.h index f74166ae..26508a44 100644 --- a/bin/pgut/pgut-fe.h +++ b/bin/pgut/pgut-fe.h @@ -57,7 +57,7 @@ typedef struct worker_conns -extern char *dbname; +extern const char *dbname; extern char *host; extern char *port; extern char *username; From 77473d99338b99c3d864afbce9c36849858f5b97 Mon Sep 17 00:00:00 2001 From: melkij Date: Thu, 19 May 2022 22:35:22 +0300 Subject: [PATCH 20/29] fix #288 Regression in #277, --parent-table refuses to work with declarative partitioned tables. Fix the logic. I didn't add a test as it will break postgresql support for all 9.x versions. --- bin/pg_repack.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/bin/pg_repack.c b/bin/pg_repack.c index 940a0d81..5bcee125 100644 --- a/bin/pg_repack.c +++ b/bin/pg_repack.c @@ -591,22 +591,26 @@ is_requested_relation_exists(char *errbuf, size_t errsize){ for (cell = table_list.head; cell; cell = cell->next) { - appendStringInfo(&sql, "($%d)", iparam + 1); + appendStringInfo(&sql, "($%d, 'r')", iparam + 1); params[iparam++] = cell->val; if (iparam < num_relations) appendStringInfoChar(&sql, ','); } for (cell = parent_table_list.head; cell; cell = cell->next) { - appendStringInfo(&sql, "($%d)", iparam + 1); + appendStringInfo(&sql, "($%d, 'p')", iparam + 1); params[iparam++] = cell->val; if (iparam < num_relations) appendStringInfoChar(&sql, ','); } appendStringInfoString(&sql, - ") AS given_t(r)" - " WHERE NOT EXISTS(" - " SELECT FROM repack.tables WHERE relid=to_regclass(given_t.r) )" + ") AS given_t(r,kind) WHERE" + /* regular --table relation or inherited --parent-table */ + " NOT EXISTS(" + " SELECT FROM repack.tables WHERE relid=to_regclass(given_t.r))" + /* declarative partitioned --parent-table */ + " AND NOT EXISTS(" + " SELECT FROM pg_catalog.pg_class c WHERE c.oid=to_regclass(given_t.r) AND c.relkind = given_t.kind AND given_t.kind = 'p')" ); /* double check the parameters array is sane */ From f42c1bd707bd5d69a9eb33494133db2e47a2c05a Mon Sep 17 00:00:00 2001 From: melkij Date: Sat, 1 Oct 2022 11:53:15 +0300 Subject: [PATCH 21/29] prepare 1.4.8 release --- META.json | 4 ++-- doc/pg_repack.rst | 8 +++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/META.json b/META.json index 54ed45c0..556447a2 100644 --- a/META.json +++ b/META.json @@ -2,7 +2,7 @@ "name": "pg_repack", "abstract": "PostgreSQL module for data reorganization", "description": "Reorganize tables in PostgreSQL databases with minimal locks", - "version": "1.4.7", + "version": "1.4.8", "maintainer": [ "Beena Emerson ", "Josh Kupershmidt ", @@ -15,7 +15,7 @@ "provides": { "pg_repack": { "file": "lib/pg_repack.sql", - "version": "1.4.7", + "version": "1.4.8", "abstract": "Reorganize tables in PostgreSQL databases with minimal locks" } }, diff --git a/doc/pg_repack.rst b/doc/pg_repack.rst index a5efc22d..87931b45 100644 --- a/doc/pg_repack.rst +++ b/doc/pg_repack.rst @@ -40,7 +40,7 @@ Requirements ------------ PostgreSQL versions - PostgreSQL 9.4, 9.5, 9.6, 10, 11, 12, 13, 14 + PostgreSQL 9.4, 9.5, 9.6, 10, 11, 12, 13, 14, 15 Disks Performing a full-table repack requires free disk space about twice as @@ -466,6 +466,12 @@ Creating indexes concurrently comes with a few caveats, please see `the document Releases -------- +* pg_repack 1.4.8 + + * Added support for PostgreSQL 15 + * Fixed --parent-table on declarative partitioned tables (issue #288) + * Removed connection info from error log (issue #285) + * pg_repack 1.4.7 * Added support for PostgreSQL 14 From b0240f884d672b9e4163097b202d1b0b2a0c2cd3 Mon Sep 17 00:00:00 2001 From: Christoph Berg Date: Wed, 28 Sep 2022 16:20:28 +0200 Subject: [PATCH 22/29] Add github workflow --- .github/workflows/regression.yml | 45 ++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 .github/workflows/regression.yml diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml new file mode 100644 index 00000000..0a6f4fe8 --- /dev/null +++ b/.github/workflows/regression.yml @@ -0,0 +1,45 @@ +name: make installcheck +on: [push, pull_request] + +jobs: + test: + strategy: + fail-fast: false + matrix: + pg: + - 15 + - 14 + - 13 + - 12 + - 11 + - 10 + + name: PostgreSQL ${{ matrix.pg }} + runs-on: ubuntu-latest + container: pgxn/pgxn-tools + steps: + + - name: Start PostgreSQL ${{ matrix.pg }} + run: pg-start ${{ matrix.pg }} + + - name: Install build-dependencies + run: apt-get install -y liblz4-dev libreadline-dev zlib1g-dev + + - name: Check out the repo + uses: actions/checkout@v2 + + - name: Put pg_repack on PATH + run: echo "$PWD/bin" >> $GITHUB_PATH + + - name: Create testts directory + run: sudo -u postgres mkdir /tmp/testts + + - name: Create testts tablespace + run: sudo -u postgres psql -c "CREATE TABLESPACE testts LOCATION '/tmp/testts'" + + - name: Test on PostgreSQL ${{ matrix.pg }} + run: pg-build-test + + - name: Show regression.diffs + if: ${{ failure() }} + run: cat regress/regression.diffs From 333e621c0d01c3a473f475789955b2316c559784 Mon Sep 17 00:00:00 2001 From: melkij Date: Mon, 3 Oct 2022 14:24:30 +0300 Subject: [PATCH 23/29] add missing build dependency libzstd-dev for github builds --- .github/workflows/regression.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 0a6f4fe8..fd4567b5 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -23,7 +23,7 @@ jobs: run: pg-start ${{ matrix.pg }} - name: Install build-dependencies - run: apt-get install -y liblz4-dev libreadline-dev zlib1g-dev + run: apt-get install -y liblz4-dev libreadline-dev zlib1g-dev libzstd-dev - name: Check out the repo uses: actions/checkout@v2 From cbe672c32da27824b5954615099c4b2123555a4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabr=C3=ADzio=20de=20Royes=20Mello?= Date: Sun, 30 Oct 2022 01:25:56 -0300 Subject: [PATCH 24/29] Fix segfault in get_order_by function When pass an invalid Oid to `get_order_by` function it lead to a segmentation full in function `get_relation_name`. Fixed if by raising an error when a table is not found. Fixes #321 --- lib/repack.c | 3 +++ regress/Makefile | 2 +- regress/expected/{issue3.out => get_order_by.out} | 9 +++++++++ regress/sql/{issue3.sql => get_order_by.sql} | 8 ++++++++ 4 files changed, 21 insertions(+), 1 deletion(-) rename regress/expected/{issue3.out => get_order_by.out} (87%) rename regress/sql/{issue3.sql => get_order_by.sql} (85%) diff --git a/lib/repack.c b/lib/repack.c index 96660c0d..7764159f 100644 --- a/lib/repack.c +++ b/lib/repack.c @@ -373,6 +373,9 @@ get_relation_name(Oid relid) char *strver; int ver; + if (!OidIsValid(nsp)) + elog(ERROR, "table name not found for OID %u", relid); + /* Get the version of the running server (PG_VERSION_NUM would return * the version we compiled the extension with) */ strver = GetConfigOptionByName("server_version_num", NULL diff --git a/regress/Makefile b/regress/Makefile index 093077f2..b8d4fba0 100644 --- a/regress/Makefile +++ b/regress/Makefile @@ -17,7 +17,7 @@ INTVERSION := $(shell echo $$(($$(echo $(VERSION).0 | sed 's/\([[:digit:]]\{1,\} # Test suite # -REGRESS := init-extension repack-setup repack-run after-schema repack-check nosuper tablespace issue3 +REGRESS := init-extension repack-setup repack-run after-schema repack-check nosuper tablespace get_order_by USE_PGXS = 1 # use pgxs if not in contrib directory PGXS := $(shell $(PG_CONFIG) --pgxs) diff --git a/regress/expected/issue3.out b/regress/expected/get_order_by.out similarity index 87% rename from regress/expected/issue3.out rename to regress/expected/get_order_by.out index edb28628..b9c2e5e0 100644 --- a/regress/expected/issue3.out +++ b/regress/expected/get_order_by.out @@ -51,3 +51,12 @@ SELECT repack.get_order_by('issue3_5_idx'::regclass::oid, 'issue3_5'::regclass:: \! pg_repack --dbname=contrib_regression --table=issue3_5 INFO: repacking table "public.issue3_5" +-- +-- pg_repack issue #321 +-- +CREATE TABLE issue321 (col1 int NOT NULL, col2 text NOT NULL); +CREATE UNIQUE INDEX issue321_idx ON issue321 (col1); +SELECT repack.get_order_by('issue321_idx'::regclass::oid, 1); +ERROR: table name not found for OID 1 +SELECT repack.get_order_by(1, 1); +ERROR: cache lookup failed for index 1 diff --git a/regress/sql/issue3.sql b/regress/sql/get_order_by.sql similarity index 85% rename from regress/sql/issue3.sql rename to regress/sql/get_order_by.sql index 9065488d..5b18e86f 100644 --- a/regress/sql/issue3.sql +++ b/regress/sql/get_order_by.sql @@ -25,3 +25,11 @@ CREATE TABLE issue3_5 (col1 int NOT NULL, col2 text NOT NULL); CREATE UNIQUE INDEX issue3_5_idx ON issue3_5 (col1 DESC NULLS FIRST, col2 COLLATE "POSIX" DESC); SELECT repack.get_order_by('issue3_5_idx'::regclass::oid, 'issue3_5'::regclass::oid); \! pg_repack --dbname=contrib_regression --table=issue3_5 + +-- +-- pg_repack issue #321 +-- +CREATE TABLE issue321 (col1 int NOT NULL, col2 text NOT NULL); +CREATE UNIQUE INDEX issue321_idx ON issue321 (col1); +SELECT repack.get_order_by('issue321_idx'::regclass::oid, 1); +SELECT repack.get_order_by(1, 1); From 6c1c8f9207828f0c8fb8c31b730519ab513f19eb Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 31 Oct 2022 10:32:09 +0100 Subject: [PATCH 25/29] Add #321 fix to news entries --- doc/pg_repack.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/pg_repack.rst b/doc/pg_repack.rst index 87931b45..ec15e0d3 100644 --- a/doc/pg_repack.rst +++ b/doc/pg_repack.rst @@ -466,6 +466,10 @@ Creating indexes concurrently comes with a few caveats, please see `the document Releases -------- +* pg_repack 1.4.9 (unreleased) + + * Fixed crash in ``get_order_by()`` using invalid relations (issue #321) + * pg_repack 1.4.8 * Added support for PostgreSQL 15 From f889f6a9d4fefdeec704cccc10bc6507255fa2a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabr=C3=ADzio=20de=20Royes=20Mello?= Date: Mon, 31 Oct 2022 11:59:37 -0300 Subject: [PATCH 26/29] Bump checkout github action version All github actions that run on Node12 are deprecated so bumped `checkout` github action version to run on Node16. https://github.blog/changelog/2022-09-22-github-actions-all-actions-will-begin-running-on-node16-instead-of-node12/ --- .github/workflows/regression.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index fd4567b5..bf997992 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -26,7 +26,7 @@ jobs: run: apt-get install -y liblz4-dev libreadline-dev zlib1g-dev libzstd-dev - name: Check out the repo - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Put pg_repack on PATH run: echo "$PWD/bin" >> $GITHUB_PATH From cf7cbbabd82017f95be3c04c703eb9c9f97178a1 Mon Sep 17 00:00:00 2001 From: Vitali Kotsik <77574919+kotik-adjust@users.noreply.github.com> Date: Thu, 23 Mar 2023 12:02:41 +0100 Subject: [PATCH 27/29] Add support for tables, which have been previously rewritten with VACUUM FULL and have Storage for all columns set to PLAIN (#337) * initial commit * Updated release notes in doc/pg_repack.rst * Delete all_fields_have_plain_storage() function. Rename repacked toast table. --------- Co-authored-by: Artur Zakirov --- doc/pg_repack.rst | 1 + lib/pgut/pgut-spi.c | 2 ++ lib/repack.c | 19 +++++++++++++++++-- regress/expected/repack-run.out | 1 + regress/expected/repack-run_1.out | 1 + regress/expected/repack-setup.out | 4 ++++ regress/sql/repack-setup.sql | 5 +++++ 7 files changed, 31 insertions(+), 2 deletions(-) diff --git a/doc/pg_repack.rst b/doc/pg_repack.rst index ec15e0d3..dd09aa92 100644 --- a/doc/pg_repack.rst +++ b/doc/pg_repack.rst @@ -469,6 +469,7 @@ Releases * pg_repack 1.4.9 (unreleased) * Fixed crash in ``get_order_by()`` using invalid relations (issue #321) + * Added support for tables that have been previously rewritten with `VACUUM FULL` and use storage=plain for all columns (issue #313) * pg_repack 1.4.8 diff --git a/lib/pgut/pgut-spi.c b/lib/pgut/pgut-spi.c index 3d77a984..9cb3c912 100644 --- a/lib/pgut/pgut-spi.c +++ b/lib/pgut/pgut-spi.c @@ -84,6 +84,8 @@ execute_with_args(int expected, const char *src, int nargs, Oid argtypes[], Datu int i; char c_nulls[FUNC_MAX_ARGS]; + memset(c_nulls, 0, sizeof(c_nulls)); + for (i = 0; i < nargs; i++) c_nulls[i] = (nulls[i] ? 'n' : ' '); diff --git a/lib/repack.c b/lib/repack.c index 7764159f..30354f93 100644 --- a/lib/repack.c +++ b/lib/repack.c @@ -922,15 +922,30 @@ repack_swap(PG_FUNCTION_ARGS) } /* swap names for toast tables and toast indexes */ - if (reltoastrelid1 == InvalidOid) + if (reltoastrelid1 == InvalidOid && reltoastrelid2 == InvalidOid) { if (reltoastidxid1 != InvalidOid || - reltoastrelid2 != InvalidOid || reltoastidxid2 != InvalidOid) elog(ERROR, "repack_swap : unexpected toast relations (T1=%u, I1=%u, T2=%u, I2=%u", reltoastrelid1, reltoastidxid1, reltoastrelid2, reltoastidxid2); /* do nothing */ } + else if (reltoastrelid1 == InvalidOid) + { + char name[NAMEDATALEN]; + + if (reltoastidxid1 != InvalidOid || + reltoastidxid2 == InvalidOid) + elog(ERROR, "repack_swap : unexpected toast relations (T1=%u, I1=%u, T2=%u, I2=%u", + reltoastrelid1, reltoastidxid1, reltoastrelid2, reltoastidxid2); + + /* rename Y to X */ + snprintf(name, NAMEDATALEN, "pg_toast_%u", oid); + RENAME_REL(reltoastrelid2, name); + snprintf(name, NAMEDATALEN, "pg_toast_%u_index", oid); + RENAME_INDEX(reltoastidxid2, name); + CommandCounterIncrement(); + } else if (reltoastrelid2 == InvalidOid) { char name[NAMEDATALEN]; diff --git a/regress/expected/repack-run.out b/regress/expected/repack-run.out index 032e47ac..79dea768 100644 --- a/regress/expected/repack-run.out +++ b/regress/expected/repack-run.out @@ -14,6 +14,7 @@ INFO: repacking table "public.tbl_gistkey" INFO: repacking table "public.tbl_idxopts" INFO: repacking table "public.tbl_only_pkey" INFO: repacking table "public.tbl_order" +INFO: repacking table "public.tbl_storage_plain" INFO: repacking table "public.tbl_with_dropped_column" INFO: repacking table "public.tbl_with_dropped_toast" INFO: repacking table "public.tbl_with_mod_column_storage" diff --git a/regress/expected/repack-run_1.out b/regress/expected/repack-run_1.out index 54efc7bb..b5dbd6e1 100644 --- a/regress/expected/repack-run_1.out +++ b/regress/expected/repack-run_1.out @@ -14,6 +14,7 @@ INFO: repacking table "public.tbl_gistkey" INFO: repacking table "public.tbl_idxopts" INFO: repacking table "public.tbl_only_pkey" INFO: repacking table "public.tbl_order" +INFO: repacking table "public.tbl_storage_plain" INFO: repacking table "public.tbl_with_dropped_column" INFO: repacking table "public.tbl_with_dropped_toast" INFO: repacking table "public.tbl_with_mod_column_storage" diff --git a/regress/expected/repack-setup.out b/regress/expected/repack-setup.out index 37028f95..254aadb8 100644 --- a/regress/expected/repack-setup.out +++ b/regress/expected/repack-setup.out @@ -70,6 +70,9 @@ CREATE TABLE tbl_with_mod_column_storage ( ); ALTER TABLE tbl_with_mod_column_storage ALTER c SET STORAGE MAIN; CREATE TABLE tbl_order (c int primary key); +CREATE TABLE tbl_storage_plain (c1 int primary key, c2 text); +ALTER TABLE tbl_storage_plain ALTER COLUMN c1 SET STORAGE PLAIN; +ALTER TABLE tbl_storage_plain ALTER COLUMN c2 SET STORAGE PLAIN; -- -- insert data -- @@ -137,3 +140,4 @@ SELECT * FROM tbl_with_dropped_toast; 2 | 20 (2 rows) +VACUUM FULL tbl_storage_plain; diff --git a/regress/sql/repack-setup.sql b/regress/sql/repack-setup.sql index f47c2ca4..350530f2 100644 --- a/regress/sql/repack-setup.sql +++ b/regress/sql/repack-setup.sql @@ -84,6 +84,10 @@ CREATE TABLE tbl_with_mod_column_storage ( ALTER TABLE tbl_with_mod_column_storage ALTER c SET STORAGE MAIN; CREATE TABLE tbl_order (c int primary key); + +CREATE TABLE tbl_storage_plain (c1 int primary key, c2 text); +ALTER TABLE tbl_storage_plain ALTER COLUMN c1 SET STORAGE PLAIN; +ALTER TABLE tbl_storage_plain ALTER COLUMN c2 SET STORAGE PLAIN; -- -- insert data -- @@ -140,3 +144,4 @@ INSERT INTO tbl_order SELECT generate_series(50, 1, -1); SELECT * FROM tbl_with_dropped_column; SELECT * FROM view_for_dropped_column; SELECT * FROM tbl_with_dropped_toast; +VACUUM FULL tbl_storage_plain; \ No newline at end of file From 652b10c769a80de07e0e0ec0cfc121a9903b613e Mon Sep 17 00:00:00 2001 From: Andreas Scherbaum Date: Thu, 23 Mar 2023 13:24:34 +0100 Subject: [PATCH 28/29] Remove outdated SPEC RPM specfiles (#340) Fix: #339 Close: #316 --- SPECS/pg_repack84.spec | 62 -------------------------------------- SPECS/pg_repack90.spec | 67 ------------------------------------------ 2 files changed, 129 deletions(-) delete mode 100644 SPECS/pg_repack84.spec delete mode 100644 SPECS/pg_repack90.spec diff --git a/SPECS/pg_repack84.spec b/SPECS/pg_repack84.spec deleted file mode 100644 index caae001c..00000000 --- a/SPECS/pg_repack84.spec +++ /dev/null @@ -1,62 +0,0 @@ -# SPEC file for pg_repack -# Copyright(C) 2009-2010 NIPPON TELEGRAPH AND TELEPHONE CORPORATION -%define sname pg_repack - -Summary: Reorganize tables in PostgreSQL databases without any locks. -Name: %{sname} -Version: 1.1.5 -Release: 1%{?dist} -License: BSD -Group: Applications/Databases -Source0: %{sname}-%{version}.tar.gz -URL: https://reorg.github.io/%{sname}/ -BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-%(%{__id_u} -n) - -BuildRequires: postgresql-devel, postgresql -Requires: postgresql, postgresql-libs - -%description -pg_repack can re-organize tables on a postgres database without any locks so that -you can retrieve or update rows in tables being reorganized. -The module is developed to be a better alternative of CLUSTER and VACUUM FULL. - -%prep -%setup -q -n %{sname}-%{version} - -%build -USE_PGXS=1 make %{?_smp_mflags} - -%install -rm -rf %{buildroot} -USE_PGXS=1 make DESTDIR=%{buildroot} - -install -d %{buildroot}%{_libdir}/pgsql -install -d %{buildroot}%{_bindir} -install -d %{buildroot}%{_datadir}/pgsql/contrib - -install -m 755 bin/pg_repack %{buildroot}%{_bindir}/pg_repack -install -m 755 lib/pg_repack.so %{buildroot}%{_libdir}/pgsql/pg_repack.so -install -m 644 lib/pg_repack.sql %{buildroot}%{_datadir}/pgsql/contrib/pg_repack.sql -install -m 644 lib/uninstall_pg_repack.sql %{buildroot}%{_datadir}/pgsql/contrib/uninstall_pg_repack.sql - -%define pg_sharedir - -%files -%defattr(755,root,root,755) -%{_bindir}/pg_repack -%{_libdir}/pgsql/pg_repack.so -%defattr(644,root,root,755) -%{_datadir}/pgsql/contrib/pg_repack.sql -%{_datadir}/pgsql/contrib/uninstall_pg_repack.sql - -%clean -rm -rf %{buildroot} - -%changelog -* Thu Oct 21 2010 - NTT OSS Center 1.1.5-1 -* Wed Sep 22 2010 - NTT OSS Center 1.1.4-1 -* Thu Apr 22 2010 - NTT OSS Center 1.1.2-1 -* Mon Jan 15 2010 - Toru SHIMOGAKI 1.0.8-1 -* Tue Sep 08 2009 - Toru SHIMOGAKI 1.0.6-1 -* Fri May 15 2009 - Toru SHIMOGAKI 1.0.4-1 -- Initial packaging diff --git a/SPECS/pg_repack90.spec b/SPECS/pg_repack90.spec deleted file mode 100644 index de56019e..00000000 --- a/SPECS/pg_repack90.spec +++ /dev/null @@ -1,67 +0,0 @@ -# SPEC file for pg_repack -# Copyright(C) 2009-2010 NIPPON TELEGRAPH AND TELEPHONE CORPORATION -%define sname pg_repack - -%define _pgdir /usr/pgsql-9.0 -%define _bindir %{_pgdir}/bin -%define _libdir %{_pgdir}/lib -%define _datadir %{_pgdir}/share - -Summary: Reorganize tables in PostgreSQL databases without any locks. -Name: %{sname} -Version: 1.1.5 -Release: 1%{?dist} -License: BSD -Group: Applications/Databases -Source0: %{sname}-%{version}.tar.gz -URL: https://reorg.github.io/%{sname}/ -BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-%(%{__id_u} -n) - -BuildRequires: postgresql90-devel, postgresql90 -Requires: postgresql90, postgresql90-libs - -%description -pg_repack can re-organize tables on a postgres database without any locks so that -you can retrieve or update rows in tables being reorganized. -The module is developed to be a better alternative of CLUSTER and VACUUM FULL. - -%prep -%setup -q -n %{sname}-%{version} - -%build -USE_PGXS=1 make %{?_smp_mflags} - -%install -rm -rf %{buildroot} -USE_PGXS=1 make DESTDIR=%{buildroot} - -install -d %{buildroot}%{_libdir} -install -d %{buildroot}%{_bindir} -install -d %{buildroot}%{_datadir}/contrib - -install -m 755 bin/pg_repack %{buildroot}%{_bindir}/pg_repack -install -m 755 lib/pg_repack.so %{buildroot}%{_libdir}/pg_repack.so -install -m 644 lib/pg_repack.sql %{buildroot}%{_datadir}/contrib/pg_repack.sql -install -m 644 lib/uninstall_pg_repack.sql %{buildroot}%{_datadir}/contrib/uninstall_pg_repack.sql - -%define pg_sharedir - -%files -%defattr(755,root,root,755) -%{_bindir}/pg_repack -%{_libdir}/pg_repack.so -%defattr(644,root,root,755) -%{_datadir}/contrib/pg_repack.sql -%{_datadir}/contrib/uninstall_pg_repack.sql - -%clean -rm -rf %{buildroot} - -%changelog -* Thu Oct 21 2010 - NTT OSS Center 1.1.5-1 -* Wed Sep 22 2010 - NTT OSS Center 1.1.4-1 -* Thu Apr 22 2010 - NTT OSS Center 1.1.2-1 -* Mon Jan 15 2010 - Toru SHIMOGAKI 1.0.8-1 -* Tue Sep 08 2009 - Toru SHIMOGAKI 1.0.6-1 -* Fri May 15 2009 - Toru SHIMOGAKI 1.0.4-1 -- Initial packaging From 7eaa988b9d4950e14b7af2f4c2b7b36624003ef6 Mon Sep 17 00:00:00 2001 From: Mathieu Agar Date: Thu, 23 Mar 2023 13:32:45 +0100 Subject: [PATCH 29/29] Documentation : the wait timeout flag (#188) * Documentation : the wait timeout flag I ran into a situation where the timeout was reached at the very beginning of the operations (because I use -D and there was an autovacuum running), and I was confused by the documentation. The details section of the documentation states clearly that the ACCESS EXCLUSIVE locks are needed at the beginning and at the end of the repack, but I propose to make the `--wait-timeout`flag documentation a little clearer on this. Thank you for this amazing tool btw ! --------- Co-authored-by: Andreas Scherbaum --- doc/pg_repack.rst | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/doc/pg_repack.rst b/doc/pg_repack.rst index dd09aa92..2aecb26e 100644 --- a/doc/pg_repack.rst +++ b/doc/pg_repack.rst @@ -197,13 +197,14 @@ Reorg Options with the ``--table`` or ``--parent-table`` options. ``-T SECS``, ``--wait-timeout=SECS`` - pg_repack needs to take an exclusive lock at the end of the - reorganization. This setting controls how many seconds pg_repack will - wait to acquire this lock. If the lock cannot be taken after this duration - and ``--no-kill-backend`` option is not specified, pg_repack will forcibly - cancel the conflicting queries. If you are using PostgreSQL version 8.4 - or newer, pg_repack will fall back to using pg_terminate_backend() to - disconnect any remaining backends after twice this timeout has passed. + pg_repack needs to take one exclusive lock at the beginning and and one + exclusive lock at the end of the reorganization. This setting controls + how many seconds pg_repack will wait to acquire this lock. If the lock + cannot be taken after this duration and ``--no-kill-backend`` option is + not specified, pg_repack will forcibly cancel the conflicting queries. + If you are using PostgreSQL version 8.4 or newer, pg_repack will fall + back to using pg_terminate_backend() to disconnect any remaining + backends after twice this timeout has passed. The default is 60 seconds. ``-D``, ``--no-kill-backend``