diff options
author | Tomas Vondra <tomas.vondra@postgresql.org> | 2023-05-17 16:49:31 +0200 |
---|---|---|
committer | Tomas Vondra <tomas.vondra@postgresql.org> | 2023-05-17 16:49:34 +0200 |
commit | 1a05c1d252993b0a59c58a6daf91a2df9333044f (patch) | |
tree | 3cc04d6bc11df32c5ed68e3a23bb42e46bfb9e92 | |
parent | 3c18d90f8907e53c3021fca13ad046133c480e4d (diff) | |
download | postgresql-1a05c1d252993b0a59c58a6daf91a2df9333044f.tar.gz |
Advance input pointer when LZ4 compressing data
LZ4File_write() did not advance the input pointer on subsequent invocations of
LZ4F_compressUpdate(). As a result the generated compressed output would be a
compressed version of the same input chunk.
Tests failed to catch this error because the data would comfortably fit
within the default buffer size, as a single chunk. Tests have been added
to provide adequate coverage of multi-chunk compression.
WriteDataToArchiveLZ4() which is also using LZ4F_compressUpdate() did
not suffer from this omission.
Author: Georgios Kokolatos <gkokolatos@pm.me>
Reported-by: Michael Paquier <michael@paquier.xyz>
Discussion: https://postgr.es/m/ZFhCyn4Gm2eu60rB%40paquier.xyz
-rw-r--r-- | src/bin/pg_dump/compress_io.h | 8 | ||||
-rw-r--r-- | src/bin/pg_dump/compress_lz4.c | 2 | ||||
-rw-r--r-- | src/bin/pg_dump/t/002_pg_dump.pl | 46 |
3 files changed, 55 insertions, 1 deletions
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h index fd8752db0d..621e03a5c8 100644 --- a/src/bin/pg_dump/compress_io.h +++ b/src/bin/pg_dump/compress_io.h @@ -17,7 +17,13 @@ #include "pg_backup_archiver.h" -/* Default size used for IO buffers */ +/* + * Default size used for IO buffers + * + * When changing this value, it's necessary to check the relevant test cases + * still exercise all the branches. This applies especially if the value is + * increased, in which case the overflow buffer may not be needed. + */ #define DEFAULT_IO_BUFFER_SIZE 4096 extern char *supports_compression(const pg_compress_specification compression_spec); diff --git a/src/bin/pg_dump/compress_lz4.c b/src/bin/pg_dump/compress_lz4.c index f97b7550d1..b869780c0b 100644 --- a/src/bin/pg_dump/compress_lz4.c +++ b/src/bin/pg_dump/compress_lz4.c @@ -588,6 +588,8 @@ LZ4Stream_write(const void *ptr, size_t size, CompressFileHandle *CFH) errno = (errno) ? errno : ENOSPC; return false; } + + ptr = ((const char *) ptr) + chunk; } return true; diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl index 93e24d5145..d66f3b42ea 100644 --- a/src/bin/pg_dump/t/002_pg_dump.pl +++ b/src/bin/pg_dump/t/002_pg_dump.pl @@ -3108,6 +3108,52 @@ my %tests = ( }, }, + 'CREATE TABLE test_compression_method' => { + create_order => 110, + create_sql => 'CREATE TABLE dump_test.test_compression_method ( + col1 text + );', + regexp => qr/^ + \QCREATE TABLE dump_test.test_compression_method (\E\n + \s+\Qcol1 text\E\n + \Q);\E + /xm, + like => { + %full_runs, + %dump_test_schema_runs, + section_pre_data => 1, + }, + unlike => { + exclude_dump_test_schema => 1, + only_dump_measurement => 1, + }, + }, + + # Insert enough data to surpass DEFAULT_IO_BUFFER_SIZE during + # (de)compression operations + 'COPY test_compression_method' => { + create_order => 111, + create_sql => 'INSERT INTO dump_test.test_compression_method (col1) ' + . 'SELECT string_agg(a::text, \'\') FROM generate_series(1,4096) a;', + regexp => qr/^ + \QCOPY dump_test.test_compression_method (col1) FROM stdin;\E + \n(?:\d{15277}\n){1}\\\.\n + /xm, + like => { + %full_runs, + data_only => 1, + section_data => 1, + only_dump_test_schema => 1, + test_schema_plus_large_objects => 1, + }, + unlike => { + binary_upgrade => 1, + exclude_dump_test_schema => 1, + schema_only => 1, + only_dump_measurement => 1, + }, + }, + 'CREATE TABLE fk_reference_test_table' => { create_order => 21, create_sql => 'CREATE TABLE dump_test.fk_reference_test_table ( |