=== Applying patches on top of PostgreSQL commit ID de65c4dade0a36026f3d0745dc8134186dab49cf === /etc/rc.d/jail: WARNING: Per-jail configuration via jail_* variables is obsolete. Please consider migrating to /etc/jail.conf. Wed Mar 26 19:17:24 UTC 2025 On branch cf/5614 nothing to commit, working tree clean === using 'git am' to apply patch ./v1-0001-Preliminary-work-to-capture-and-expose-separate-r.patch === Applying: Preliminary work to capture and expose separate record (leaf page) and metadata (non-leaf page) index access statistics in the system views, with partial coverage of B-Trees. Using index info to reconstruct a base tree... M contrib/amcheck/verify_nbtree.c M contrib/bloom/blinsert.c M contrib/bloom/blscan.c M contrib/bloom/blutils.c M contrib/pageinspect/rawpage.c M contrib/pg_prewarm/autoprewarm.c M contrib/pg_surgery/heap_surgery.c M contrib/pg_visibility/pg_visibility.c M contrib/pgstattuple/pgstattuple.c M doc/src/sgml/monitoring.sgml M src/backend/access/brin/brin.c M src/backend/access/gin/ginget.c M src/backend/access/gin/ginutil.c M src/backend/access/gist/gist.c M src/backend/access/gist/gistget.c M src/backend/access/gist/gistutil.c M src/backend/access/gist/gistvacuum.c M src/backend/access/hash/hash.c M src/backend/access/heap/heapam.c M src/backend/access/heap/heapam_handler.c M src/backend/access/heap/vacuumlazy.c M src/backend/access/nbtree/nbtinsert.c M src/backend/access/nbtree/nbtree.c M src/backend/access/nbtree/nbtsearch.c M src/backend/access/nbtree/nbtutils.c M src/backend/access/spgist/spgscan.c M src/backend/access/spgist/spgutils.c M src/backend/access/spgist/spgvacuum.c M src/backend/catalog/system_views.sql M src/backend/storage/aio/read_stream.c M src/backend/storage/buffer/bufmgr.c M src/backend/utils/adt/pgstatfuncs.c M src/include/access/nbtree.h M src/include/catalog/pg_proc.dat M src/include/pgstat.h M src/include/storage/bufmgr.h M src/test/regress/expected/rules.out Falling back to patching base and 3-way merge... Auto-merging src/test/regress/expected/rules.out Auto-merging src/include/storage/bufmgr.h Auto-merging src/include/pgstat.h Auto-merging src/include/catalog/pg_proc.dat Auto-merging src/include/access/nbtree.h Auto-merging src/backend/utils/adt/pgstatfuncs.c Auto-merging src/backend/storage/buffer/bufmgr.c CONFLICT (content): Merge conflict in src/backend/storage/buffer/bufmgr.c Auto-merging src/backend/storage/aio/read_stream.c Auto-merging src/backend/catalog/system_views.sql Auto-merging src/backend/access/spgist/spgvacuum.c CONFLICT (content): Merge conflict in src/backend/access/spgist/spgvacuum.c Auto-merging src/backend/access/spgist/spgutils.c Auto-merging src/backend/access/spgist/spgscan.c Auto-merging src/backend/access/nbtree/nbtutils.c Auto-merging src/backend/access/nbtree/nbtsearch.c Auto-merging src/backend/access/nbtree/nbtree.c CONFLICT (content): Merge conflict in src/backend/access/nbtree/nbtree.c Auto-merging src/backend/access/nbtree/nbtinsert.c Auto-merging src/backend/access/heap/vacuumlazy.c Auto-merging src/backend/access/heap/heapam_handler.c CONFLICT (content): Merge conflict in src/backend/access/heap/heapam_handler.c Auto-merging src/backend/access/heap/heapam.c Auto-merging src/backend/access/hash/hash.c Auto-merging src/backend/access/gist/gistvacuum.c CONFLICT (content): Merge conflict in src/backend/access/gist/gistvacuum.c Auto-merging src/backend/access/gist/gistutil.c Auto-merging src/backend/access/gist/gistget.c Auto-merging src/backend/access/gist/gist.c Auto-merging src/backend/access/gin/ginutil.c Auto-merging src/backend/access/gin/ginget.c Auto-merging src/backend/access/brin/brin.c Auto-merging doc/src/sgml/monitoring.sgml Auto-merging contrib/pgstattuple/pgstattuple.c Auto-merging contrib/pg_visibility/pg_visibility.c Auto-merging contrib/pg_surgery/heap_surgery.c Auto-merging contrib/pg_prewarm/autoprewarm.c Auto-merging contrib/pageinspect/rawpage.c Auto-merging contrib/bloom/blutils.c Auto-merging contrib/bloom/blscan.c Auto-merging contrib/bloom/blinsert.c Auto-merging contrib/amcheck/verify_nbtree.c error: Failed to merge in the changes. hint: Use 'git am --show-current-patch=diff' to see the failed patch Patch failed at 0001 Preliminary work to capture and expose separate record (leaf page) and metadata (non-leaf page) index access statistics in the system views, with partial coverage of B-Trees. When you have resolved this problem, run "git am --continue". If you prefer to skip this patch, run "git am --skip" instead. To restore the original branch and stop patching, run "git am --abort". Unstaged changes after reset: M contrib/amcheck/verify_heapam.c M contrib/amcheck/verify_nbtree.c M contrib/bloom/blinsert.c M contrib/bloom/blscan.c M contrib/bloom/blutils.c M contrib/bloom/blvacuum.c M contrib/pageinspect/btreefuncs.c M contrib/pageinspect/rawpage.c M contrib/pg_prewarm/autoprewarm.c M contrib/pg_surgery/heap_surgery.c M contrib/pg_visibility/pg_visibility.c M contrib/pgstattuple/pgstatapprox.c M contrib/pgstattuple/pgstatindex.c M contrib/pgstattuple/pgstattuple.c M doc/src/sgml/monitoring.sgml M src/backend/access/brin/brin.c M src/backend/access/brin/brin_pageops.c M src/backend/access/brin/brin_revmap.c M src/backend/access/gin/ginbtree.c M src/backend/access/gin/ginfast.c M src/backend/access/gin/ginget.c M src/backend/access/gin/ginutil.c M src/backend/access/gin/ginvacuum.c M src/backend/access/gist/gist.c M src/backend/access/gist/gistbuild.c M src/backend/access/gist/gistget.c M src/backend/access/gist/gistutil.c M src/backend/access/gist/gistvacuum.c M src/backend/access/hash/hash.c M src/backend/access/hash/hashpage.c M src/backend/access/heap/heapam.c M src/backend/access/heap/heapam_handler.c M src/backend/access/heap/hio.c M src/backend/access/heap/vacuumlazy.c M src/backend/access/heap/visibilitymap.c M src/backend/access/nbtree/nbtinsert.c M src/backend/access/nbtree/nbtpage.c M src/backend/access/nbtree/nbtree.c M src/backend/access/nbtree/nbtsearch.c M src/backend/access/nbtree/nbtutils.c M src/backend/access/spgist/spgdoinsert.c M src/backend/access/spgist/spgscan.c M src/backend/access/spgist/spgutils.c M src/backend/access/spgist/spgvacuum.c M src/backend/access/transam/xloginsert.c M src/backend/catalog/system_views.sql M src/backend/commands/sequence.c M src/backend/storage/aio/read_stream.c M src/backend/storage/buffer/bufmgr.c M src/backend/storage/freespace/freespace.c M src/backend/utils/activity/pgstat_database.c M src/backend/utils/activity/pgstat_relation.c M src/backend/utils/adt/pgstatfuncs.c M src/include/access/nbtree.h M src/include/catalog/pg_proc.dat M src/include/pgstat.h M src/include/storage/bufmgr.h M src/test/regress/expected/rules.out === using patch(1) to apply patch ./v1-0001-Preliminary-work-to-capture-and-expose-separate-r.patch === patch: unrecognized option `--no-backup-if-mismatch' usage: patch [-bCcEeflNnRstuv] [-B backup-prefix] [-D symbol] [-d directory] [-F max-fuzz] [-i patchfile] [-o out-file] [-p strip-count] [-r rej-name] [-V t | nil | never | none] [-x number] [-z backup-ext] [--posix] [origfile [patchfile]] patch strategy, NULL); + ++>>>>>>> theirs /* * We are not going to stay here for a long time, aggressively grab an * exclusive lock. diff --cc src/backend/access/heap/heapam_handler.c index 24d3765aa20,12167d9bb59..00000000000 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@@ -2128,49 -2123,194 +2129,134 @@@ heapam_scan_bitmap_next_tuple(TableScan { BitmapHeapScanDesc bscan = (BitmapHeapScanDesc) scan; HeapScanDesc hscan = (HeapScanDesc) bscan; ++<<<<<<< ours + OffsetNumber targoffset; + Page page; + ItemId lp; ++======= + BlockNumber block; + Buffer buffer; + Snapshot snapshot; + int ntup; + TBMIterateResult *tbmres; + + Assert(scan->rs_flags & SO_TYPE_BITMAPSCAN); + + hscan->rs_cindex = 0; + hscan->rs_ntuples = 0; + + *blockno = InvalidBlockNumber; + *recheck = true; + + do + { + CHECK_FOR_INTERRUPTS(); + + tbmres = tbm_iterate(&scan->st.rs_tbmiterator); + + if (tbmres == NULL) + return false; + + /* + * Ignore any claimed entries past what we think is the end of the + * relation. It may have been extended after the start of our scan (we + * only hold an AccessShareLock, and it could be inserts from this + * backend). We don't take this optimization in SERIALIZABLE + * isolation though, as we need to examine all invisible tuples + * reachable by the index. + */ + } while (!IsolationIsSerializable() && + tbmres->blockno >= hscan->rs_nblocks); + + /* Got a valid block */ + *blockno = tbmres->blockno; + *recheck = tbmres->recheck; + + /* + * We can skip fetching the heap page if we don't need any fields from the + * heap, the bitmap entries don't need rechecking, and all tuples on the + * page are visible to our transaction. + */ + if (!(scan->rs_flags & SO_NEED_TUPLES) && + !tbmres->recheck && + VM_ALL_VISIBLE(scan->rs_rd, tbmres->blockno, &bscan->rs_vmbuffer)) + { + /* can't be lossy in the skip_fetch case */ + Assert(tbmres->ntuples >= 0); + Assert(bscan->rs_empty_tuples_pending >= 0); + + bscan->rs_empty_tuples_pending += tbmres->ntuples; + + return true; + } + + block = tbmres->blockno; + + /* + * Acquire pin on the target heap page, trading in any pin we held before. + */ + hscan->rs_cbuf = ReleaseAndReadBuffer(hscan->rs_cbuf, + scan->rs_rd, + block, + NULL); + hscan->rs_cblock = block; + buffer = hscan->rs_cbuf; + snapshot = scan->rs_snapshot; + + ntup = 0; + + /* + * Prune and repair fragmentation for the whole page, if possible. + */ + heap_page_prune_opt(scan->rs_rd, buffer); + + /* + * We must hold share lock on the buffer content while examining tuple + * visibility. Afterwards, however, the tuples we have found to be + * visible are guaranteed good as long as we hold the buffer pin. + */ + LockBuffer(buffer, BUFFER_LOCK_SHARE); ++>>>>>>> theirs /* - * We need two separate strategies for lossy and non-lossy cases. + * Out of range? If so, nothing more to look at on this page */ - if (tbmres->ntuples >= 0) + while (hscan->rs_cindex >= hscan->rs_ntuples) { /* - * Bitmap is non-lossy, so we just look through the offsets listed in - * tbmres; but we have to follow any HOT chain starting at each such - * offset. + * Emit empty tuples before advancing to the next block */ - int curslot; - - for (curslot = 0; curslot < tbmres->ntuples; curslot++) + if (bscan->rs_empty_tuples_pending > 0) { - OffsetNumber offnum = tbmres->offsets[curslot]; - ItemPointerData tid; - HeapTupleData heapTuple; - - ItemPointerSet(&tid, block, offnum); - if (heap_hot_search_buffer(&tid, scan->rs_rd, buffer, snapshot, - &heapTuple, NULL, true)) - hscan->rs_vistuples[ntup++] = ItemPointerGetOffsetNumber(&tid); - } - } - else - { - /* - * Bitmap is lossy, so we must examine each line pointer on the page. - * But we can ignore HOT chains, since we'll check each tuple anyway. - */ - Page page = BufferGetPage(buffer); - OffsetNumber maxoff = PageGetMaxOffsetNumber(page); - OffsetNumber offnum; - - for (offnum = FirstOffsetNumber; offnum <= maxoff; offnum = OffsetNumberNext(offnum)) - { - ItemId lp; - HeapTupleData loctup; - bool valid; + /* + * If we don't have to fetch the tuple, just return nulls. + */ + ExecStoreAllNullTuple(slot); + bscan->rs_empty_tuples_pending--; - lp = PageGetItemId(page, offnum); - if (!ItemIdIsNormal(lp)) - continue; - loctup.t_data = (HeapTupleHeader) PageGetItem(page, lp); - loctup.t_len = ItemIdGetLength(lp); - loctup.t_tableOid = scan->rs_rd->rd_id; - ItemPointerSet(&loctup.t_self, block, offnum); - valid = HeapTupleSatisfiesVisibility(&loctup, snapshot, buffer); - if (valid) - { - hscan->rs_vistuples[ntup++] = offnum; - PredicateLockTID(scan->rs_rd, &loctup.t_self, snapshot, - HeapTupleHeaderGetXmin(loctup.t_data)); - } - HeapCheckForSerializableConflictOut(valid, scan->rs_rd, &loctup, - buffer, snapshot); + /* + * We do not recheck all NULL tuples. Because the streaming read + * API only yields TBMIterateResults for blocks actually fetched + * from the heap, we must unset `recheck` ourselves here to ensure + * correct results. + * + * Our read stream callback accrues a count of empty tuples to + * emit and then emits them after emitting tuples from the next + * fetched block. If no blocks need fetching, we'll emit the + * accrued count at the end of the scan. + */ + *recheck = false; + return true; } - } - - LockBuffer(buffer, BUFFER_LOCK_UNLOCK); - - Assert(ntup <= MaxHeapTuplesPerPage); - hscan->rs_ntuples = ntup; - - if (tbmres->ntuples >= 0) - (*exact_pages)++; - else - (*lossy_pages)++; - - /* - * Return true to indicate that a valid block was found and the bitmap is - * not exhausted. If there are no visible tuples on this page, - * hscan->rs_ntuples will be 0 and heapam_scan_bitmap_next_tuple() will - * return false returning control to this function to advance to the next - * block in the bitmap. - */ - return true; -} - -static bool -heapam_scan_bitmap_next_tuple(TableScanDesc scan, - TupleTableSlot *slot) -{ - BitmapHeapScanDesc bscan = (BitmapHeapScanDesc) scan; - HeapScanDesc hscan = (HeapScanDesc) bscan; - OffsetNumber targoffset; - Page page; - ItemId lp; - if (bscan->rs_empty_tuples_pending > 0) - { /* - * If we don't have to fetch the tuple, just return nulls. + * Returns false if the bitmap is exhausted and there are no further + * blocks we need to scan. */ - ExecStoreAllNullTuple(slot); - bscan->rs_empty_tuples_pending--; - return true; + if (!BitmapHeapScanNextBlock(scan, recheck, lossy_pages, exact_pages)) + return false; } - /* - * Out of range? If so, nothing more to look at on this page - */ - if (hscan->rs_cindex >= hscan->rs_ntuples) - return false; - targoffset = hscan->rs_vistuples[hscan->rs_cindex]; page = BufferGetPage(hscan->rs_cbuf); lp = PageGetItemId(page, targoffset); diff --cc src/backend/access/nbtree/nbtree.c index 80b04d6ca2a,7aa24e1f12b..00000000000 --- a/src/backend/access/nbtree/nbtree.c +++ b/src/backend/access/nbtree/nbtree.c @@@ -1177,6 -1136,17 +1177,20 @@@ backtrack attempt_pagedel = false; backtrack_to = P_NONE; ++<<<<<<< ours ++======= + /* call vacuum_delay_point while not holding any buffer lock */ + vacuum_delay_point(false); + + /* + * We can't use _bt_getbuf() here because it always applies + * _bt_checkpage(), which will barf on an all-zero page. We want to + * recycle all-zero pages, not fail. Also, we want to use a nondefault + * buffer access strategy. + */ + buf = ReadBufferExtended(rel, MAIN_FORKNUM, blkno, RBM_NORMAL, + info->strategy, NULL); ++>>>>>>> theirs _bt_lockbuf(rel, buf, BT_READ); page = BufferGetPage(buf); opaque = NULL; diff --cc src/backend/access/spgist/spgvacuum.c index 77deb226b7e,b4cf3470b65..00000000000 --- a/src/backend/access/spgist/spgvacuum.c +++ b/src/backend/access/spgist/spgvacuum.c @@@ -619,12 -618,17 +619,20 @@@ vacuumRedirectAndPlaceholder(Relation i * Process one page during a bulkdelete scan */ static void -spgvacuumpage(spgBulkDeleteState *bds, BlockNumber blkno) +spgvacuumpage(spgBulkDeleteState *bds, Buffer buffer) { Relation index = bds->info->index; - Buffer buffer; + BlockNumber blkno = BufferGetBlockNumber(buffer); Page page; ++<<<<<<< ours ++======= + /* call vacuum_delay_point while not holding any buffer lock */ + vacuum_delay_point(false); + + buffer = ReadBufferExtended(index, MAIN_FORKNUM, blkno, + RBM_NORMAL, bds->info->strategy, NULL); ++>>>>>>> theirs LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); page = (Page) BufferGetPage(buffer); diff --cc src/backend/storage/buffer/bufmgr.c index 323382dcfa8,6acb8089a52..00000000000 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@@ -1253,12 -1263,12 +1260,16 @@@ StartReadBuffersImpl(ReadBuffersOperati BlockNumber blockNum, int *nblocks, int flags, ++<<<<<<< ours + bool allow_forwarding) ++======= + bool *hit) ++>>>>>>> theirs { int actual_nblocks = *nblocks; - int io_buffers_len = 0; int maxcombine = 0; + Assert(*nblocks == 1 || allow_forwarding); Assert(*nblocks > 0); Assert(*nblocks <= MAX_IO_COMBINE_LIMIT); @@@ -1266,52 -1276,14 +1277,63 @@@ { bool found; ++<<<<<<< ours + if (allow_forwarding && buffers[i] != InvalidBuffer) + { + BufferDesc *bufHdr; + + /* + * This is a buffer that was pinned by an earlier call to + * StartReadBuffers(), but couldn't be handled in one operation at + * that time. The operation was split, and the caller has passed + * an already pinned buffer back to us to handle the rest of the + * operation. It must continue at the expected block number. + */ + Assert(BufferGetBlockNumber(buffers[i]) == blockNum + i); + + /* + * It might be an already valid buffer (a hit) that followed the + * final contiguous block of an earlier I/O (a miss) marking the + * end of it, or a buffer that some other backend has since made + * valid by performing the I/O for us, in which case we can handle + * it as a hit now. It is safe to check for a BM_VALID flag with + * a relaxed load, because we got a fresh view of it while pinning + * it in the previous call. + * + * On the other hand if we don't see BM_VALID yet, it must be an + * I/O that was split by the previous call and we need to try to + * start a new I/O from this block. We're also racing against any + * other backend that might start the I/O or even manage to mark + * it BM_VALID after this check, but StartBufferIO() will handle + * those cases. + */ + if (BufferIsLocal(buffers[i])) + bufHdr = GetLocalBufferDescriptor(-buffers[i] - 1); + else + bufHdr = GetBufferDescriptor(buffers[i] - 1); + Assert(pg_atomic_read_u32(&bufHdr->state) & BM_TAG_VALID); + found = pg_atomic_read_u32(&bufHdr->state) & BM_VALID; + } + else + { + buffers[i] = PinBufferForBlock(operation->rel, + operation->smgr, + operation->persistence, + operation->forknum, + blockNum + i, + operation->strategy, + &found); + } ++======= + buffers[i] = PinBufferForBlock(operation->rel, + operation->smgr, + operation->persistence, + operation->forknum, + blockNum + i, + operation->strategy, + &found, + hit); ++>>>>>>> theirs if (found) { @@@ -1420,10 -1374,10 +1442,15 @@@ StartReadBuffers(ReadBuffersOperation * Buffer *buffers, BlockNumber blockNum, int *nblocks, - int flags) + int flags, + bool *hit) { ++<<<<<<< ours + return StartReadBuffersImpl(operation, buffers, blockNum, nblocks, flags, + true /* expect forwarded buffers */ ); ++======= + return StartReadBuffersImpl(operation, buffers, blockNum, nblocks, flags, hit); ++>>>>>>> theirs } /* @@@ -1443,8 -1395,7 +1471,12 @@@ StartReadBuffer(ReadBuffersOperation *o int nblocks = 1; bool result; ++<<<<<<< ours + result = StartReadBuffersImpl(operation, buffer, blocknum, &nblocks, flags, + false /* single block, no forwarding */ ); ++======= + result = StartReadBuffersImpl(operation, buffer, blocknum, &nblocks, flags, hit); ++>>>>>>> theirs Assert(nblocks == 1); /* single block can't be short */ return result;