From 7879efdd807eee84477c1e63fdedb922b23f8420 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Tue, 18 Oct 2022 18:16:15 +0300 Subject: [PATCH 1/4] Treat empty tables the same as non-empty tables Currently there is a bug when there are empty tables added to the schema. The dump file then contains an empty COPY statement that is currently mishandled and causes the script to skip to count 9999 --- pgtricks/pg_dump_splitsort.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pgtricks/pg_dump_splitsort.py b/pgtricks/pg_dump_splitsort.py index aab1258..c79bb1f 100755 --- a/pgtricks/pg_dump_splitsort.py +++ b/pgtricks/pg_dump_splitsort.py @@ -11,7 +11,7 @@ from pgtricks.mergesort import MergeSort -COPY_RE = re.compile(r'COPY .*? \(.*?\) FROM stdin;\n$') +COPY_RE = re.compile(r'COPY .*? FROM stdin;\n$') KIBIBYTE, MEBIBYTE, GIBIBYTE = 2**10, 2**20, 2**30 MEMORY_UNITS = {"": 1, "k": KIBIBYTE, "m": MEBIBYTE, "g": GIBIBYTE} From 5fb94c603dc67d289a569f87ca91b91aabc99591 Mon Sep 17 00:00:00 2001 From: Antti Kaihola <13725+akaihola@users.noreply.github.com> Date: Fri, 19 Apr 2024 22:09:58 +0300 Subject: [PATCH 2/4] Add test for `COPY_RE` --- pgtricks/tests/test_pg_dump_splitsort.py | 29 +++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/pgtricks/tests/test_pg_dump_splitsort.py b/pgtricks/tests/test_pg_dump_splitsort.py index 74e6b56..e38e8d3 100644 --- a/pgtricks/tests/test_pg_dump_splitsort.py +++ b/pgtricks/tests/test_pg_dump_splitsort.py @@ -3,7 +3,34 @@ import pytest -from pgtricks.pg_dump_splitsort import linecomp, memory_size, split_sql_file, try_float +from pgtricks.pg_dump_splitsort import ( + COPY_RE, + linecomp, + memory_size, + split_sql_file, + try_float, +) + + +@pytest.mark.parametrize( + ("test_input", "expected"), + [ + ("COPY table_name (column1, column2) FROM stdin;\n", True), + ("COPY table_name (column1, column2) FROM stdin;\n", True), + ("COPY table_name FROM stdin;\n", True), + ("COPY table_name FROM stdin;\n", True), + ("COPYtable_name FROM stdin;\n", False), # No space after COPY + ("COPY table_name FROMstdin;\n", False), # No space before stdin + ("COPY table_name FROM ;\n", False), # Missing stdin + ("COPY table_name stdin;\n", False), # Missing FROM + ("COPY FROM stdin;\n", False), # Missing table name + ], +) +def test_sql_copy_regular_expression(test_input, expected): + """Test that `COPY_RE` matches/doesn't match the expected strings.""" + result = COPY_RE.match(test_input) is not None + + assert result == expected @pytest.mark.parametrize( From 7ff68e5dd18c0ad227c15db7063052f4407f89aa Mon Sep 17 00:00:00 2001 From: Antti Kaihola <13725+akaihola@users.noreply.github.com> Date: Fri, 19 Apr 2024 22:10:08 +0300 Subject: [PATCH 3/4] Fix `COPY_RE` for empty tables and multi-space --- pgtricks/pg_dump_splitsort.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pgtricks/pg_dump_splitsort.py b/pgtricks/pg_dump_splitsort.py index c79bb1f..0b9eee7 100755 --- a/pgtricks/pg_dump_splitsort.py +++ b/pgtricks/pg_dump_splitsort.py @@ -11,7 +11,7 @@ from pgtricks.mergesort import MergeSort -COPY_RE = re.compile(r'COPY .*? FROM stdin;\n$') +COPY_RE = re.compile(r"COPY\s+\S+\s+(\(.*?\)\s+)?FROM\s+stdin;\n$") KIBIBYTE, MEBIBYTE, GIBIBYTE = 2**10, 2**20, 2**30 MEMORY_UNITS = {"": 1, "k": KIBIBYTE, "m": MEBIBYTE, "g": GIBIBYTE} From 09d178280fd34803970699a0914deb88fd8bbefe Mon Sep 17 00:00:00 2001 From: Antti Kaihola <13725+akaihola@users.noreply.github.com> Date: Sat, 20 Apr 2024 11:38:38 +0300 Subject: [PATCH 4/4] Update the change log --- CHANGES.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index c3796a7..6c8946f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -12,7 +12,7 @@ Removed Fixed ----- - +- Empty tables are now handled correctly. - Very large tables are now sorted without crashing. This is done by merge sorting in temporary files. @@ -37,7 +37,7 @@ Added Added ----- -Document ``pg_incremental_backup.py`` in the README file +- Document ``pg_incremental_backup.py`` in the README file 0.9_ / 2015-03-10