=== Applying patches on top of PostgreSQL commit ID 122a9af5def2db78f2c2131958eab8873bfee93b === /etc/rc.d/jail: WARNING: Per-jail configuration via jail_* variables is obsolete. Please consider migrating to /etc/jail.conf. Tue Mar 18 09:22:26 UTC 2025 On branch cf/4351 nothing to commit, working tree clean === using 'git am' to apply patch ./v20241106-0001-WIP-index-batching-prefetching.patch === Applying: WIP: index batching / prefetching Using index info to reconstruct a base tree... M src/backend/access/heap/heapam_handler.c M src/backend/access/index/genam.c M src/backend/access/index/indexam.c M src/backend/executor/execIndexing.c M src/backend/executor/execReplication.c M src/backend/executor/nodeIndexonlyscan.c M src/backend/executor/nodeIndexscan.c M src/backend/utils/adt/selfuncs.c M src/backend/utils/misc/guc_tables.c M src/backend/utils/misc/postgresql.conf.sample M src/include/access/amapi.h M src/include/access/genam.h M src/include/access/relscan.h M src/include/nodes/execnodes.h M src/test/regress/expected/sysviews.out M src/tools/pgindent/typedefs.list Falling back to patching base and 3-way merge... Auto-merging src/tools/pgindent/typedefs.list Auto-merging src/test/regress/expected/sysviews.out CONFLICT (content): Merge conflict in src/test/regress/expected/sysviews.out Auto-merging src/include/nodes/execnodes.h Auto-merging src/include/access/relscan.h CONFLICT (content): Merge conflict in src/include/access/relscan.h Auto-merging src/include/access/genam.h CONFLICT (content): Merge conflict in src/include/access/genam.h Auto-merging src/include/access/amapi.h Auto-merging src/backend/utils/misc/postgresql.conf.sample Auto-merging src/backend/utils/misc/guc_tables.c Auto-merging src/backend/utils/adt/selfuncs.c CONFLICT (content): Merge conflict in src/backend/utils/adt/selfuncs.c Auto-merging src/backend/executor/nodeIndexscan.c CONFLICT (content): Merge conflict in src/backend/executor/nodeIndexscan.c Auto-merging src/backend/executor/nodeIndexonlyscan.c CONFLICT (content): Merge conflict in src/backend/executor/nodeIndexonlyscan.c Auto-merging src/backend/executor/execReplication.c CONFLICT (content): Merge conflict in src/backend/executor/execReplication.c Auto-merging src/backend/executor/execIndexing.c CONFLICT (content): Merge conflict in src/backend/executor/execIndexing.c Auto-merging src/backend/access/index/indexam.c CONFLICT (content): Merge conflict in src/backend/access/index/indexam.c Auto-merging src/backend/access/index/genam.c CONFLICT (content): Merge conflict in src/backend/access/index/genam.c Auto-merging src/backend/access/heap/heapam_handler.c CONFLICT (content): Merge conflict in src/backend/access/heap/heapam_handler.c error: Failed to merge in the changes. hint: Use 'git am --show-current-patch=diff' to see the failed patch Patch failed at 0001 WIP: index batching / prefetching When you have resolved this problem, run "git am --continue". If you prefer to skip this patch, run "git am --skip" instead. To restore the original branch and stop patching, run "git am --abort". Unstaged changes after reset: M src/backend/access/heap/heapam_handler.c M src/backend/access/index/genam.c M src/backend/access/index/indexam.c M src/backend/executor/execIndexing.c M src/backend/executor/execReplication.c M src/backend/executor/nodeIndexonlyscan.c M src/backend/executor/nodeIndexscan.c M src/backend/utils/adt/selfuncs.c M src/backend/utils/misc/guc_tables.c M src/backend/utils/misc/postgresql.conf.sample M src/include/access/amapi.h M src/include/access/genam.h M src/include/access/relscan.h M src/include/nodes/execnodes.h M src/test/regress/expected/sysviews.out M src/tools/pgindent/typedefs.list === using patch(1) to apply patch ./v20241106-0001-WIP-index-batching-prefetching.patch === patch: unrecognized option `--no-backup-if-mismatch' usage: patch [-bCcEeflNnRstuv] [-B backup-prefix] [-D symbol] [-d directory] [-F max-fuzz] [-i patchfile] [-o out-file] [-p strip-count] [-r rej-name] [-V t | nil | never | none] [-x number] [-z backup-ext] [--posix] [origfile [patchfile]] patch >>>>>> theirs index_rescan(indexScan, NULL, 0, NULL, 0); } else diff --cc src/backend/access/index/genam.c index 8f532e14590,3a9d2d483d7..00000000000 --- a/src/backend/access/index/genam.c +++ b/src/backend/access/index/genam.c @@@ -446,12 -445,20 +446,26 @@@ systable_beginscan(Relation heapRelatio elog(ERROR, "column is not in index"); } + /* + * No batching/prefetch for catalogs. We don't expect that to help + * very much, because we usually need just one row, and even if we + * need multiple rows, they tend to be colocated in heap. + * + * XXX Maybe we could do that, the prefetching only ramps up over + * time. But then we need to be careful about infinite recursion when + * looking up effective_io_concurrency for a tablespace in the + * catalog. + */ sysscan->iscan = index_beginscan(heapRelation, irel, ++<<<<<<< ours + snapshot, NULL, nkeys, 0); ++======= + snapshot, nkeys, 0, false); ++>>>>>>> theirs index_rescan(sysscan->iscan, idxkey, nkeys, NULL, 0); sysscan->scan = NULL; + + pfree(idxkey); } else { @@@ -707,8 -718,17 +721,21 @@@ systable_beginscan_ordered(Relation hea elog(ERROR, "column is not in index"); } + /* + * No batching/prefetch for catalogs. We don't expect that to help very + * much, because we usually need just one row, and even if we need + * multiple rows, they tend to be colocated in heap. + * + * XXX Maybe we could do that, the prefetching only ramps up over time. + * But then we need to be careful about infinite recursion when looking up + * effective_io_concurrency for a tablespace in the catalog. + */ sysscan->iscan = index_beginscan(heapRelation, indexRelation, ++<<<<<<< ours + snapshot, NULL, nkeys, 0); ++======= + snapshot, nkeys, 0, false); ++>>>>>>> theirs index_rescan(sysscan->iscan, idxkey, nkeys, NULL, 0); sysscan->scan = NULL; diff --cc src/backend/access/index/indexam.c index 55ec4c10352,2849ab97cdf..00000000000 --- a/src/backend/access/index/indexam.c +++ b/src/backend/access/index/indexam.c @@@ -256,8 -270,8 +270,13 @@@ IndexScanDes index_beginscan(Relation heapRelation, Relation indexRelation, Snapshot snapshot, ++<<<<<<< ours + IndexScanInstrumentation *instrument, + int nkeys, int norderbys) ++======= + int nkeys, int norderbys, + bool enable_batching) ++>>>>>>> theirs { IndexScanDesc scan; @@@ -579,10 -601,9 +646,16 @@@ index_parallelrescan(IndexScanDesc scan * Caller must be holding suitable locks on the heap and the index. */ IndexScanDesc ++<<<<<<< ours +index_beginscan_parallel(Relation heaprel, Relation indexrel, + IndexScanInstrumentation *instrument, + int nkeys, int norderbys, + ParallelIndexScanDesc pscan) ++======= + index_beginscan_parallel(Relation heaprel, Relation indexrel, int nkeys, + int norderbys, ParallelIndexScanDesc pscan, + bool enable_batching) ++>>>>>>> theirs { Snapshot snapshot; IndexScanDesc scan; diff --cc src/backend/executor/execIndexing.c index e3fe9b78bb5,742a963bc29..00000000000 --- a/src/backend/executor/execIndexing.c +++ b/src/backend/executor/execIndexing.c @@@ -816,7 -809,12 +816,16 @@@ check_exclusion_or_unique_constraint(Re retry: conflict = false; found_self = false; ++<<<<<<< ours + index_scan = index_beginscan(heap, index, &DirtySnapshot, NULL, indnkeyatts, 0); ++======= + + /* + * XXX Does not seem useful to do prefetching for checks of constraints. + * We would probably need just the first item anyway. + */ + index_scan = index_beginscan(heap, index, &DirtySnapshot, indnkeyatts, 0, false); ++>>>>>>> theirs index_rescan(index_scan, scankeys, indnkeyatts, NULL, 0); while (index_getnext_slot(index_scan, ForwardScanDirection, existing_slot)) diff --cc src/backend/executor/execReplication.c index 0a9b880d250,6be3744361d..00000000000 --- a/src/backend/executor/execReplication.c +++ b/src/backend/executor/execReplication.c @@@ -201,8 -244,13 +201,18 @@@ RelationFindReplTupleByIndex(Relation r /* Build scan key. */ skey_attoff = build_replindex_scan_key(skey, rel, idxrel, searchslot); ++<<<<<<< ours + /* Start an index scan. */ + scan = index_beginscan(rel, idxrel, &snap, NULL, skey_attoff, 0); ++======= + /* + * Start an index scan. + * + * XXX No prefetching for replication identity. We expect to find just one + * row, so prefetching is pointless. + */ + scan = index_beginscan(rel, idxrel, &snap, skey_attoff, 0, false); ++>>>>>>> theirs retry: found = false; diff --cc src/backend/executor/nodeIndexonlyscan.c index f464cca9507,c030c0df6fe..00000000000 --- a/src/backend/executor/nodeIndexonlyscan.c +++ b/src/backend/executor/nodeIndexonlyscan.c @@@ -92,9 -97,9 +97,10 @@@ IndexOnlyNext(IndexOnlyScanState *node scandesc = index_beginscan(node->ss.ss_currentRelation, node->ioss_RelationDesc, estate->es_snapshot, + &node->ioss_Instrument, node->ioss_NumScanKeys, - node->ioss_NumOrderByKeys); + node->ioss_NumOrderByKeys, + node->ioss_CanBatch); node->ioss_ScanDesc = scandesc; @@@ -773,23 -781,22 +821,33 @@@ ExecIndexOnlyScanInitializeDSM(IndexOnl index_parallelscan_initialize(node->ss.ss_currentRelation, node->ioss_RelationDesc, estate->es_snapshot, - piscan); + instrument, parallel_aware, pcxt->nworkers, + &node->ioss_SharedInfo, piscan); shm_toc_insert(pcxt->toc, node->ss.ps.plan->plan_node_id, piscan); ++<<<<<<< ours + if (!parallel_aware) + { + /* Only here to initialize SharedInfo in DSM */ + return; + } + ++======= + /* + * XXX do we actually want prefetching for parallel index scans? Maybe + * not, but then we need to be careful not to call index_batch_getnext_tid + * (which now can happen, because we'll call IndexOnlyNext even for + * parallel plans). + */ ++>>>>>>> theirs node->ioss_ScanDesc = index_beginscan_parallel(node->ss.ss_currentRelation, node->ioss_RelationDesc, + &node->ioss_Instrument, node->ioss_NumScanKeys, node->ioss_NumOrderByKeys, - piscan); + piscan, + node->ioss_CanBatch); node->ioss_ScanDesc->xs_want_itup = true; node->ioss_VMBuffer = InvalidBuffer; @@@ -839,23 -837,14 +897,28 @@@ ExecIndexOnlyScanInitializeWorker(Index piscan = shm_toc_lookup(pwcxt->toc, node->ss.ps.plan->plan_node_id, false); ++<<<<<<< ours + if (instrument) + node->ioss_SharedInfo = (SharedIndexScanInstrumentation *) + OffsetToPointer(piscan, piscan->ps_offset_ins); + + if (!parallel_aware) + { + /* Only here to set up worker node's SharedInfo */ + return; + } + ++======= + /* XXX do we actually want prefetching for parallel index scans? */ ++>>>>>>> theirs node->ioss_ScanDesc = index_beginscan_parallel(node->ss.ss_currentRelation, node->ioss_RelationDesc, + &node->ioss_Instrument, node->ioss_NumScanKeys, node->ioss_NumOrderByKeys, - piscan); + piscan, + node->ioss_CanBatch); node->ioss_ScanDesc->xs_want_itup = true; /* @@@ -868,24 -857,33 +931,56 @@@ node->ioss_OrderByKeys, node->ioss_NumOrderByKeys); } ++<<<<<<< ours +/* ---------------------------------------------------------------- + * ExecIndexOnlyScanRetrieveInstrumentation + * + * Transfer index-only scan statistics from DSM to private memory. + * ---------------------------------------------------------------- + */ +void +ExecIndexOnlyScanRetrieveInstrumentation(IndexOnlyScanState *node) +{ + SharedIndexScanInstrumentation *SharedInfo = node->ioss_SharedInfo; + size_t size; + + if (SharedInfo == NULL) + return; + + /* Create a copy of SharedInfo in backend-local memory */ + size = offsetof(SharedIndexScanInstrumentation, winstrument) + + SharedInfo->num_workers * sizeof(IndexScanInstrumentation); + node->ioss_SharedInfo = palloc(size); + memcpy(node->ioss_SharedInfo, SharedInfo, size); ++======= + /* + * ios_prefetch_block + * Callback to only prefetch blocks that are not all-visible. + * + * We don't want to inspect the visibility map repeatedly, so the result of + * VM_ALL_VISIBLE is stored in the batch private data. The values are set + * to 0 by default, so we use two constants to remember if all-visible or + * not all-visible. + */ + static bool + ios_prefetch_block(IndexScanDesc scan, void *arg, int index) + { + IndexOnlyScanState *node = (IndexOnlyScanState *) arg; + + if (scan->xs_batch->privateData[index] == IOS_UNKNOWN_VISIBILITY) + { + bool all_visible; + ItemPointer tid = &scan->xs_batch->heaptids[index]; + + all_visible = VM_ALL_VISIBLE(scan->heapRelation, + ItemPointerGetBlockNumber(tid), + &node->ioss_VMBuffer); + + scan->xs_batch->privateData[index] + = all_visible ? IOS_ALL_VISIBLE : IOS_NOT_ALL_VISIBLE; + } + + /* prefetch only blocks that are not all-visible */ + return (scan->xs_batch->privateData[index] == IOS_NOT_ALL_VISIBLE); ++>>>>>>> theirs } diff --cc src/backend/executor/nodeIndexscan.c index 7fcaa37fe62,8bbd3606566..00000000000 --- a/src/backend/executor/nodeIndexscan.c +++ b/src/backend/executor/nodeIndexscan.c @@@ -109,9 -109,9 +109,10 @@@ IndexNext(IndexScanState *node scandesc = index_beginscan(node->ss.ss_currentRelation, node->iss_RelationDesc, estate->es_snapshot, + &node->iss_Instrument, node->iss_NumScanKeys, - node->iss_NumOrderByKeys); + node->iss_NumOrderByKeys, + node->iss_CanBatch); node->iss_ScanDesc = scandesc; @@@ -205,9 -207,9 +208,10 @@@ IndexNextWithReorder(IndexScanState *no scandesc = index_beginscan(node->ss.ss_currentRelation, node->iss_RelationDesc, estate->es_snapshot, + &node->iss_Instrument, node->iss_NumScanKeys, - node->iss_NumOrderByKeys); + node->iss_NumOrderByKeys, + false); node->iss_ScanDesc = scandesc; @@@ -1709,23 -1686,19 +1727,30 @@@ ExecIndexScanInitializeDSM(IndexScanSta index_parallelscan_initialize(node->ss.ss_currentRelation, node->iss_RelationDesc, estate->es_snapshot, - piscan); + instrument, parallel_aware, pcxt->nworkers, + &node->iss_SharedInfo, piscan); shm_toc_insert(pcxt->toc, node->ss.ps.plan->plan_node_id, piscan); ++<<<<<<< ours + if (!parallel_aware) + { + /* Only here to initialize SharedInfo in DSM */ + return; + } + ++======= + /* + * XXX do we actually want prefetching for parallel index scans? + */ ++>>>>>>> theirs node->iss_ScanDesc = index_beginscan_parallel(node->ss.ss_currentRelation, node->iss_RelationDesc, + &node->iss_Instrument, node->iss_NumScanKeys, node->iss_NumOrderByKeys, - piscan); + piscan, + node->iss_CanBatch); /* * If no run-time keys to calculate or they are ready, go ahead and pass @@@ -1773,23 -1737,16 +1798,30 @@@ ExecIndexScanInitializeWorker(IndexScan piscan = shm_toc_lookup(pwcxt->toc, node->ss.ps.plan->plan_node_id, false); ++<<<<<<< ours + if (instrument) + node->iss_SharedInfo = (SharedIndexScanInstrumentation *) + OffsetToPointer(piscan, piscan->ps_offset_ins); + + if (!parallel_aware) + { + /* Only here to set up worker node's SharedInfo */ + return; + } + ++======= + /* + * XXX do we actually want prefetching for parallel index scans? + */ ++>>>>>>> theirs node->iss_ScanDesc = index_beginscan_parallel(node->ss.ss_currentRelation, node->iss_RelationDesc, + &node->iss_Instrument, node->iss_NumScanKeys, node->iss_NumOrderByKeys, - piscan); + piscan, + node->iss_CanBatch); /* * If no run-time keys to calculate or they are ready, go ahead and pass diff --cc src/backend/utils/adt/selfuncs.c index 5b35debc8ff,02ea0eea149..00000000000 --- a/src/backend/utils/adt/selfuncs.c +++ b/src/backend/utils/adt/selfuncs.c @@@ -6550,9 -6343,14 +6550,19 @@@ get_actual_variable_endpoint(Relation h InitNonVacuumableSnapshot(SnapshotNonVacuumable, GlobalVisTestFor(heapRel)); + /* + * XXX I'm not sure about batching/prefetching here. In most cases we + * expect to find the endpoints immediately, but sometimes we have a lot + * of dead tuples - and then prefetching might help. + */ index_scan = index_beginscan(heapRel, indexRel, ++<<<<<<< ours + &SnapshotNonVacuumable, NULL, + 1, 0); ++======= + &SnapshotNonVacuumable, + 1, 0, false); ++>>>>>>> theirs /* Set it up for index-only scan */ index_scan->xs_want_itup = true; index_rescan(index_scan, scankeys, 1, NULL, 0); diff --cc src/include/access/genam.h index 5b2ab181b5f,1d9a0868a9b..00000000000 --- a/src/include/access/genam.h +++ b/src/include/access/genam.h @@@ -14,7 -14,7 +14,11 @@@ #ifndef GENAM_H #define GENAM_H ++<<<<<<< ours +#include "access/htup.h" ++======= + #include "access/itup.h" ++>>>>>>> theirs #include "access/sdir.h" #include "access/skey.h" #include "nodes/tidbitmap.h" @@@ -178,11 -159,10 +185,16 @@@ extern void index_insert_cleanup(Relati extern IndexScanDesc index_beginscan(Relation heapRelation, Relation indexRelation, Snapshot snapshot, ++<<<<<<< ours + IndexScanInstrumentation *instrument, + int nkeys, int norderbys); ++======= + int nkeys, int norderbys, + bool enable_batching); ++>>>>>>> theirs extern IndexScanDesc index_beginscan_bitmap(Relation indexRelation, Snapshot snapshot, + IndexScanInstrumentation *instrument, int nkeys); extern void index_rescan(IndexScanDesc scan, ScanKey keys, int nkeys, @@@ -202,10 -177,9 +214,16 @@@ extern void index_parallelscan_initiali ParallelIndexScanDesc target); extern void index_parallelrescan(IndexScanDesc scan); extern IndexScanDesc index_beginscan_parallel(Relation heaprel, ++<<<<<<< ours + Relation indexrel, + IndexScanInstrumentation *instrument, + int nkeys, int norderbys, + ParallelIndexScanDesc pscan); ++======= + Relation indexrel, int nkeys, int norderbys, + ParallelIndexScanDesc pscan, + bool enable_batching); ++>>>>>>> theirs extern ItemPointer index_getnext_tid(IndexScanDesc scan, ScanDirection direction); struct TupleTableSlot; diff --cc src/include/access/relscan.h index b5e0fb386c0,8fd2da8514d..00000000000 --- a/src/include/access/relscan.h +++ b/src/include/access/relscan.h @@@ -123,7 -129,8 +123,12 @@@ typedef struct IndexFetchTableDat Relation rel; } IndexFetchTableData; ++<<<<<<< ours +struct IndexScanInstrumentation; ++======= + /* Forward declaration, the prefetch callback needs IndexScanDescData. */ + typedef struct IndexScanBatchData IndexScanBatchData; ++>>>>>>> theirs /* * We use the same IndexScanDescData structure for both amgettuple-based diff --cc src/test/regress/expected/sysviews.out index 83228cfca29,14b38ed4d46..00000000000 --- a/src/test/regress/expected/sysviews.out +++ b/src/test/regress/expected/sysviews.out @@@ -172,7 -171,7 +173,11 @@@ select name, setting from pg_settings w enable_seqscan | on enable_sort | on enable_tidscan | on ++<<<<<<< ours +(24 rows) ++======= + (23 rows) ++>>>>>>> theirs -- There are always wait event descriptions for various types. InjectionPoint -- may be present or absent, depending on history since last postmaster start.