=== Applying patches on top of PostgreSQL commit ID 972c14fb9134fdfd76ea6ebcf98a55a945bbc988 === /etc/rc.d/jail: WARNING: Per-jail configuration via jail_* variables is obsolete. Please consider migrating to /etc/jail.conf. Wed Apr 15 07:01:35 UTC 2026 On branch cf/5621 nothing to commit, working tree clean === using 'git am' to apply patch ./v6-0001-Add-whole-record-WAL-compression-alongside-FPI-co.patch === Applying: Add whole-record WAL compression alongside FPI compression Using index info to reconstruct a base tree... M src/backend/access/transam/xlog.c M src/backend/access/transam/xloginsert.c M src/backend/access/transam/xlogreader.c M src/backend/utils/misc/guc_parameters.dat M src/backend/utils/misc/postgresql.conf.sample M src/include/access/xlog.h M src/include/access/xlogreader.h M src/include/utils/guc_hooks.h M src/test/perl/PostgreSQL/Test/Cluster.pm M src/test/recovery/t/046_checkpoint_logical_slot.pl Falling back to patching base and 3-way merge... Auto-merging src/test/recovery/t/046_checkpoint_logical_slot.pl Auto-merging src/test/perl/PostgreSQL/Test/Cluster.pm Auto-merging src/include/utils/guc_hooks.h Auto-merging src/include/access/xlogreader.h Auto-merging src/include/access/xlog.h CONFLICT (content): Merge conflict in src/include/access/xlog.h Auto-merging src/backend/utils/misc/postgresql.conf.sample Auto-merging src/backend/utils/misc/guc_parameters.dat Auto-merging src/backend/access/transam/xlogreader.c Auto-merging src/backend/access/transam/xloginsert.c CONFLICT (content): Merge conflict in src/backend/access/transam/xloginsert.c Auto-merging src/backend/access/transam/xlog.c CONFLICT (content): Merge conflict in src/backend/access/transam/xlog.c error: Failed to merge in the changes. hint: Use 'git am --show-current-patch=diff' to see the failed patch Patch failed at 0001 Add whole-record WAL compression alongside FPI compression When you have resolved this problem, run "git am --continue". If you prefer to skip this patch, run "git am --skip" instead. To restore the original branch and stop patching, run "git am --abort". === using patch(1) to apply patch ./v6-0001-Add-whole-record-WAL-compression-alongside-FPI-co.patch === patching file src/backend/access/transam/xlog.c Hunk #1 succeeded at 142 (offset 4 lines). Hunk #2 succeeded at 752 with fuzz 2 (offset 34 lines). Hunk #3 succeeded at 1084 (offset 34 lines). patching file src/backend/access/transam/xloginsert.c Hunk #1 succeeded at 39 with fuzz 1. Hunk #2 succeeded at 65 (offset 1 line). Hunk #3 succeeded at 104 (offset 1 line). Hunk #4 FAILED at 134. Hunk #5 succeeded at 156 (offset 1 line). Hunk #6 succeeded at 177 (offset 1 line). Hunk #7 succeeded at 259 (offset 1 line). Hunk #8 succeeded at 492 (offset 4 lines). Hunk #9 succeeded at 625 (offset 4 lines). Hunk #10 succeeded at 670 (offset 4 lines). Hunk #11 succeeded at 689 (offset 4 lines). Hunk #12 succeeded at 809 with fuzz 1 (offset 55 lines). Hunk #13 succeeded at 909 (offset 55 lines). Hunk #14 succeeded at 941 (offset 55 lines). Hunk #15 succeeded at 967 (offset 55 lines). Hunk #16 succeeded at 1056 (offset 55 lines). Hunk #17 succeeded at 1152 (offset 55 lines). Hunk #18 succeeded at 1169 (offset 55 lines). Hunk #19 succeeded at 1288 (offset 55 lines). Hunk #20 succeeded at 1311 (offset 55 lines). Hunk #21 succeeded at 1700 (offset 27 lines). 1 out of 21 hunks FAILED -- saving rejects to file src/backend/access/transam/xloginsert.c.rej patching file src/backend/access/transam/xlogreader.c Hunk #2 succeeded at 56 (offset 1 line). Hunk #3 succeeded at 174 (offset 1 line). Hunk #4 succeeded at 539 (offset 1 line). Hunk #5 succeeded at 651 (offset 1 line). Hunk #6 succeeded at 726 (offset 1 line). Hunk #7 succeeded at 749 (offset 1 line). Hunk #8 succeeded at 765 (offset 1 line). Hunk #9 succeeded at 798 (offset 1 line). Hunk #10 succeeded at 839 (offset 1 line). Hunk #11 succeeded at 896 (offset 1 line). Hunk #12 succeeded at 907 (offset 1 line). Hunk #13 succeeded at 926 (offset 1 line). Hunk #14 succeeded at 938 (offset 1 line). Hunk #15 succeeded at 961 (offset 1 line). Hunk #16 succeeded at 1768 (offset 19 lines). Hunk #17 succeeded at 1910 (offset 19 lines). Hunk #18 succeeded at 2092 (offset 19 lines). patching file src/backend/utils/misc/guc_parameters.dat Hunk #1 succeeded at 3449 (offset 126 lines). patching file src/backend/utils/misc/postgresql.conf.sample Hunk #1 succeeded at 261 (offset 16 lines). patching file src/include/access/xlog.h Hunk #1 FAILED at 57. 1 out of 1 hunk FAILED -- saving rejects to file src/include/access/xlog.h.rej patching file src/include/access/xlogreader.h patching file src/include/access/xlogrecord.h patching file src/include/utils/guc_hooks.h Hunk #1 succeeded at 175 (offset 6 lines). patching file src/test/perl/PostgreSQL/Test/Cluster.pm Hunk #2 succeeded at 3187 (offset 35 lines). patching file src/test/recovery/Makefile patching file src/test/recovery/t/046_checkpoint_logical_slot.pl Hunk #1 succeeded at 115 (offset -1 lines). patching file src/test/recovery/t/052_wal_compression.pl patching file src/test/recovery/wal_compression.conf Unstaged changes after reset: M src/backend/access/transam/xlog.c M src/backend/access/transam/xloginsert.c M src/backend/access/transam/xlogreader.c M src/backend/utils/misc/guc_parameters.dat M src/backend/utils/misc/postgresql.conf.sample M src/include/access/xlogreader.h M src/include/access/xlogrecord.h M src/include/utils/guc_hooks.h M src/test/perl/PostgreSQL/Test/Cluster.pm M src/test/recovery/Makefile M src/test/recovery/t/046_checkpoint_logical_slot.pl Removing src/backend/access/transam/xloginsert.c.rej Removing src/include/access/xlog.h.rej Removing src/test/recovery/t/052_wal_compression.pl Removing src/test/recovery/wal_compression.conf === using 'git apply' to apply patch ./v6-0001-Add-whole-record-WAL-compression-alongside-FPI-co.patch === Applied patch to 'src/backend/access/transam/xlog.c' with conflicts. Applied patch to 'src/backend/access/transam/xloginsert.c' with conflicts. Applied patch to 'src/backend/access/transam/xlogreader.c' cleanly. Applied patch to 'src/backend/utils/misc/guc_parameters.dat' cleanly. Applied patch to 'src/backend/utils/misc/postgresql.conf.sample' cleanly. Applied patch to 'src/include/access/xlog.h' with conflicts. Applied patch to 'src/include/access/xlogreader.h' cleanly. Applied patch to 'src/include/access/xlogrecord.h' cleanly. Applied patch to 'src/include/utils/guc_hooks.h' cleanly. Applied patch to 'src/test/perl/PostgreSQL/Test/Cluster.pm' cleanly. Applied patch to 'src/test/recovery/Makefile' cleanly. Applied patch to 'src/test/recovery/t/046_checkpoint_logical_slot.pl' cleanly. Falling back to direct application... Falling back to direct application... U src/backend/access/transam/xlog.c U src/backend/access/transam/xloginsert.c U src/include/access/xlog.h diff --cc src/backend/access/transam/xlog.c index f85b5286086,0e5f96c5b57..00000000000 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@@ -749,7 -718,20 +750,24 @@@ static void WALInsertLockAcquireExclusi static void WALInsertLockRelease(void); static void WALInsertLockUpdateInsertingAt(XLogRecPtr insertingAt); ++<<<<<<< ours +static void XLogChecksums(uint32 new_type); ++======= + #ifdef WAL_DEBUG + /* Read length of a record, accounting for possible compression */ + static uint32 + XLogGetRecordTotalLen(XLogRecord *record) + { + if (record->xl_info & XLR_COMPRESSED) + { + XLogCompressionHeader *c = (XLogCompressionHeader *) record; + Assert(c->decompressed_length > 0); + return c->decompressed_length; + } + return record->xl_tot_len; + } + #endif ++>>>>>>> theirs /* * Insert an XLOG record represented by an already-constructed chain of data diff --cc src/backend/access/transam/xloginsert.c index f2e10b82b7d,16da53ba565..00000000000 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@@ -39,9 -39,9 +39,10 @@@ #include "replication/origin.h" #include "storage/bufmgr.h" #include "storage/proc.h" + #include "utils/guc_hooks.h" #include "utils/memutils.h" #include "utils/pgstat_internal.h" +#include "utils/rel.h" /* * Guess the maximum buffer size required to store a compressed version of @@@ -116,14 -134,62 +135,67 @@@ static uint8 curinsert_flags = 0 static XLogRecData hdr_rdt; static char *hdr_scratch = NULL; ++<<<<<<< ours +#define SizeOfXlogOrigin (sizeof(ReplOriginId) + sizeof(char)) +#define SizeOfXLogTransactionId (sizeof(TransactionId) + sizeof(char)) ++======= + /* + * GUC: Maximum memory per backend for WAL compression. + * + * Controls the size of the shared FPI compression buffer as well as the + * threshold for whole-record compression: records larger than this value + * will not be compressed as a whole (per-FPI compression still applies). + * + * The default equals MIN_WAL_COMPRESSION_BUFFER, which is the total memory + * previously used by embedded per-block compressed_page arrays. + * + * Actual memory consumption per backend is approximately 2x this value + * because we need both an input buffer and an output buffer for whole-record + * compression. + */ + int wal_compression_buffer; ++>>>>>>> theirs - #define HEADER_SCRATCH_SIZE \ - (SizeOfXLogRecord + \ - MaxSizeOfXLogRecordBlockHeader * (XLR_MAX_BLOCK_ID + 1) + \ - SizeOfXLogRecordDataHeaderLong + SizeOfXlogOrigin + \ - SizeOfXLogTransactionId) + static XLogRecData compressed_rdt_hdr; + + /* + * Compression buffers, allocated once and grown with repalloc if + * wal_compression_buffer increases. compression_buffers_size tracks the + * wal_compression_buffer value at which they were last allocated. + * + * compression_buf is a single wal_compression_buffer-sized staging area + * shared between two mutually exclusive uses: + * + * - FPI compression (skip_fpi_compression = false): compressed block images + * are packed into compression_buf one after another, with + * compression_buf_offset tracking the fill level. rdt nodes point + * directly into this buffer. + * + * - Whole-record compression (skip_fpi_compression = true): XLogCompressRdt + * flattens the rdt chain into compression_buf before compressing into + * compressed_data. FPI compression is skipped in this path, so the two + * uses never overlap. + * + * compressed_data holds the compressor output for whole-record compression. + * Its size is the maximum compressed output for wal_compression_buffer bytes. + */ + static char *compression_buf = NULL; + static int compression_buf_offset; /* fill level for FPI packing */ + static char *compressed_data = NULL; + static uint32 compressed_data_size; /* allocated size of compressed_data */ + static int compression_buffers_size; /* wal_compression_buffer at last alloc */ + + /* + * In assert builds (where MEMORY_CONTEXT_CHECKING is active), verify every + * palloc sentinel in the process after operations that write to compression + * buffers. This catches overflows that corrupt adjacent palloc blocks. + * Compiles to nothing in non-assert builds. + */ + #ifdef MEMORY_CONTEXT_CHECKING + #define CheckCompressionMemory() MemoryContextCheck(TopMemoryContext) + #else + #define CheckCompressionMemory() ((void) 0) + #endif /* * An array of XLogRecData structs, to hold registered data. @@@ -552,54 -803,84 +812,135 @@@ XLogSimpleInsertInt64(RmgrId rmid, uint } /* ++<<<<<<< ours + * XLogGetFakeLSN - get a fake LSN for an index page that isn't WAL-logged. + * + * Some index AMs use LSNs to detect concurrent page modifications, but not + * all index pages are WAL-logged. This function provides a sequence of fake + * LSNs for that purpose. + */ +XLogRecPtr +XLogGetFakeLSN(Relation rel) +{ + if (rel->rd_rel->relpersistence == RELPERSISTENCE_TEMP) + { + /* + * Temporary relations are only accessible in our session, so a simple + * backend-local counter will do. + */ + static XLogRecPtr counter = FirstNormalUnloggedLSN; + + return counter++; + } + else if (rel->rd_rel->relpersistence == RELPERSISTENCE_UNLOGGED) + { + /* + * Unlogged relations are accessible from other backends, and survive + * (clean) restarts. GetFakeLSNForUnloggedRel() handles that for us. + */ + return GetFakeLSNForUnloggedRel(); + } + else + { + /* + * WAL-logging on this relation will start after commit, so its LSNs + * must be distinct numbers smaller than the LSN at the next commit. + * Emit a dummy WAL record if insert-LSN hasn't advanced after the + * last call. + */ + static XLogRecPtr lastlsn = InvalidXLogRecPtr; + XLogRecPtr currlsn = GetXLogInsertEndRecPtr(); + + Assert(!RelationNeedsWAL(rel)); + Assert(RelationIsPermanent(rel)); + + /* No need for an actual record if we already have a distinct LSN */ + if (XLogRecPtrIsValid(lastlsn) && lastlsn == currlsn) + currlsn = XLogAssignLSN(); + + lastlsn = currlsn; + return currlsn; + } ++======= + * Allocate (or grow) the two compression buffers. + * + * Must be called outside a critical section. InitXLogInsert() calls this at + * backend start when wal_compression is enabled. The GUC assign hooks for + * wal_compression and wal_compression_buffer call it when those GUCs change + * at runtime, which also always happens outside critical sections. + * + * If the buffers are already sized to wal_compression_buffer this is a no-op. + */ + static void + AllocCompressionBuffers(void) + { + uint32 new_size = wal_compression_buffer; + uint32 compressed_buf_size; + + Assert(CritSectionCount == 0); + Assert(xloginsert_cxt != NULL); + + if (compression_buffers_size >= new_size) + return; /* already big enough */ + + compressed_buf_size = PGLZ_MAX_OUTPUT(new_size); + #ifdef USE_LZ4 + compressed_buf_size = Max(compressed_buf_size, + LZ4_COMPRESSBOUND(new_size)); + #endif + #ifdef USE_ZSTD + compressed_buf_size = Max(compressed_buf_size, + ZSTD_COMPRESSBOUND(new_size)); + #endif + compressed_buf_size += SizeOfXLogCompressedRecord; + + if (compression_buf == NULL) + { + compression_buf = MemoryContextAlloc(xloginsert_cxt, new_size); + compressed_data = MemoryContextAlloc(xloginsert_cxt, compressed_buf_size); + } + else + { + compression_buf = repalloc(compression_buf, new_size); + compressed_data = repalloc(compressed_data, compressed_buf_size); + } + + compressed_data_size = compressed_buf_size; + compression_buffers_size = new_size; + CheckCompressionMemory(); + } + + /* + * GUC assign hook for wal_compression. + * + * Allocates compression buffers immediately when compression is first enabled + * so that XLogInsert() never needs to allocate inside a critical section. + * If xloginsert_cxt is not yet set up (very early startup), the allocation is + * deferred to InitXLogInsert(). + */ + void + assign_wal_compression(int newval, void *extra) + { + if (newval != WAL_COMPRESSION_NONE && + xloginsert_cxt != NULL && + compression_buf == NULL) + AllocCompressionBuffers(); + } + + /* + * GUC assign hook for wal_compression_buffer. + * + * Grows the compression buffers immediately when the GUC is increased so that + * XLogInsert() never needs to repalloc inside a critical section. + */ + void + assign_wal_compression_buffer(int newval, void *extra) + { + if (wal_compression != WAL_COMPRESSION_NONE && + xloginsert_cxt != NULL && + compression_buf != NULL) + AllocCompressionBuffers(); ++>>>>>>> theirs } /* diff --cc src/include/access/xlog.h index 437b4f32349,a6e759847f2..00000000000 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@@ -57,7 -57,8 +57,12 @@@ extern PGDLLIMPORT int CommitDelay extern PGDLLIMPORT int CommitSiblings; extern PGDLLIMPORT bool track_wal_io_timing; extern PGDLLIMPORT int wal_decode_buffer_size; ++<<<<<<< ours +extern PGDLLIMPORT int data_checksums; ++======= + extern PGDLLIMPORT int wal_compression_threshold; + extern PGDLLIMPORT int wal_compression_buffer; ++>>>>>>> theirs extern PGDLLIMPORT int CheckPointSegments;