=== Applying patches on top of PostgreSQL commit ID daa16893faa96246d758eb2cc27e2f75c5308296 === /etc/rc.d/jail: WARNING: Per-jail configuration via jail_* variables is obsolete. Please consider migrating to /etc/jail.conf. Fri Apr 4 07:56:18 UTC 2025 On branch cf/4884 nothing to commit, working tree clean === using 'git am' to apply patch ./0001-Fix-rare-recovery-shutdown-hang-due-to-checkpointer.patch === Applying: Fix rare recovery shutdown hang due to checkpointer. Using index info to reconstruct a base tree... M src/backend/postmaster/postmaster.c Falling back to patching base and 3-way merge... Auto-merging src/backend/postmaster/postmaster.c CONFLICT (content): Merge conflict in src/backend/postmaster/postmaster.c error: Failed to merge in the changes. hint: Use 'git am --show-current-patch=diff' to see the failed patch Patch failed at 0001 Fix rare recovery shutdown hang due to checkpointer. When you have resolved this problem, run "git am --continue". If you prefer to skip this patch, run "git am --skip" instead. To restore the original branch and stop patching, run "git am --abort". Unstaged changes after reset: M src/backend/postmaster/postmaster.c === using patch(1) to apply patch ./0001-Fix-rare-recovery-shutdown-hang-due-to-checkpointer.patch === patch: unrecognized option `--no-backup-if-mismatch' usage: patch [-bCcEeflNnRstuv] [-B backup-prefix] [-D symbol] [-d directory] [-F max-fuzz] [-i patchfile] [-o out-file] [-p strip-count] [-r rej-name] [-V t | nil | never | none] [-x number] [-z backup-ext] [--posix] [origfile [patchfile]] patch NoShutdown); - - /* Waken archiver for the last time */ - if (PgArchPID != 0) - signal_child(PgArchPID, SIGUSR2); - - /* - * Waken walsenders for the last time. No regular backends - * should be around anymore. - */ - SignalChildren(SIGUSR2); - - pmState = PM_SHUTDOWN_2; - } - else - { - /* - * Any unexpected exit of the checkpointer (including FATAL - * exit) is treated as a crash. - */ - HandleChildCrash(pid, exitstatus, - _("checkpointer process")); - } - - continue; - } - - /* - * Was it the wal writer? Normal exit can be ignored; we'll start a - * new one at the next iteration of the postmaster's main loop, if - * necessary. Any other exit condition is treated as a crash. - */ - if (pid == WalWriterPID) - { - WalWriterPID = 0; - if (!EXIT_STATUS_0(exitstatus)) - HandleChildCrash(pid, exitstatus, - _("WAL writer process")); - continue; - } - - /* - * Was it the wal receiver? If exit status is zero (normal) or one - * (FATAL exit), we assume everything is all right just like normal - * backends. (If we need a new wal receiver, we'll start one at the - * next iteration of the postmaster's main loop.) - */ - if (pid == WalReceiverPID) - { - WalReceiverPID = 0; - if (!EXIT_STATUS_0(exitstatus) && !EXIT_STATUS_1(exitstatus)) - HandleChildCrash(pid, exitstatus, - _("WAL receiver process")); - continue; - } - - /* - * Was it the wal summarizer? Normal exit can be ignored; we'll start - * a new one at the next iteration of the postmaster's main loop, if - * necessary. Any other exit condition is treated as a crash. - */ - if (pid == WalSummarizerPID) - { - WalSummarizerPID = 0; - if (!EXIT_STATUS_0(exitstatus)) - HandleChildCrash(pid, exitstatus, - _("WAL summarizer process")); - continue; - } - - /* - * Was it the autovacuum launcher? Normal exit can be ignored; we'll - * start a new one at the next iteration of the postmaster's main - * loop, if necessary. Any other exit condition is treated as a - * crash. - */ - if (pid == AutoVacPID) - { - AutoVacPID = 0; - if (!EXIT_STATUS_0(exitstatus)) - HandleChildCrash(pid, exitstatus, - _("autovacuum launcher process")); - continue; - } - - /* - * Was it the archiver? If exit status is zero (normal) or one (FATAL - * exit), we assume everything is all right just like normal backends - * and just try to restart a new one so that we immediately retry - * archiving remaining files. (If fail, we'll try again in future - * cycles of the postmaster's main loop.) Unless we were waiting for - * it to shut down; don't restart it in that case, and - * PostmasterStateMachine() will advance to the next shutdown step. - */ - if (pid == PgArchPID) - { - PgArchPID = 0; - if (!EXIT_STATUS_0(exitstatus) && !EXIT_STATUS_1(exitstatus)) - HandleChildCrash(pid, exitstatus, - _("archiver process")); - if (PgArchStartupAllowed()) - PgArchPID = StartChildProcess(ArchiverProcess); - continue; - } - - /* Was it the system logger? If so, try to start a new one */ - if (pid == SysLoggerPID) - { - SysLoggerPID = 0; - /* for safety's sake, launch new logger *first* */ - SysLoggerPID = SysLogger_Start(); - if (!EXIT_STATUS_0(exitstatus)) - LogChildExit(LOG, _("system logger process"), - pid, exitstatus); - continue; - } - - /* - * Was it the slot sync worker? Normal exit or FATAL exit can be - * ignored (FATAL can be caused by libpqwalreceiver on receiving - * shutdown request by the startup process during promotion); we'll - * start a new one at the next iteration of the postmaster's main - * loop, if necessary. Any other exit condition is treated as a crash. - */ - if (pid == SlotSyncWorkerPID) - { - SlotSyncWorkerPID = 0; - if (!EXIT_STATUS_0(exitstatus) && !EXIT_STATUS_1(exitstatus)) - HandleChildCrash(pid, exitstatus, - _("slot sync worker process")); - continue; - } - - /* Was it one of our background workers? */ - if (CleanupBackgroundWorker(pid, exitstatus)) - { - /* have it be restarted */ - HaveCrashedWorker = true; - continue; - } - - /* - * Else do standard backend child cleanup. - */ - CleanupBackend(pid, exitstatus); - } /* loop over pending child-death reports */ - - /* - * After cleaning out the SIGCHLD queue, see if we have any state changes - * or actions to make. - */ - PostmasterStateMachine(); -} - -/* - * Scan the bgworkers list and see if the given PID (which has just stopped - * or crashed) is in it. Handle its shutdown if so, and return true. If not a - * bgworker, return false. - * - * This is heavily based on CleanupBackend. One important difference is that - * we don't know yet that the dying process is a bgworker, so we must be silent - * until we're sure it is. - */ -static bool -CleanupBackgroundWorker(int pid, - int exitstatus) /* child's exit status */ -{ - char namebuf[MAXPGPATH]; - slist_mutable_iter iter; - - slist_foreach_modify(iter, &BackgroundWorkerList) - { - RegisteredBgWorker *rw; - - rw = slist_container(RegisteredBgWorker, rw_lnode, iter.cur); - - if (rw->rw_pid != pid) - continue; - -#ifdef WIN32 - /* see CleanupBackend */ - if (exitstatus == ERROR_WAIT_NO_CHILDREN) - exitstatus = 0; -#endif - - snprintf(namebuf, MAXPGPATH, _("background worker \"%s\""), - rw->rw_worker.bgw_type); - - - if (!EXIT_STATUS_0(exitstatus)) - { - /* Record timestamp, so we know when to restart the worker. */ - rw->rw_crashed_at = GetCurrentTimestamp(); - } - else - { - /* Zero exit status means terminate */ - rw->rw_crashed_at = 0; - rw->rw_terminate = true; - } - - /* - * Additionally, just like a backend, any exit status other than 0 or - * 1 is considered a crash and causes a system-wide restart. - */ - if (!EXIT_STATUS_0(exitstatus) && !EXIT_STATUS_1(exitstatus)) - { - HandleChildCrash(pid, exitstatus, namebuf); - return true; - } - - /* - * We must release the postmaster child slot. If the worker failed to - * do so, it did not clean up after itself, requiring a crash-restart - * cycle. - */ - if (!ReleasePostmasterChildSlot(rw->rw_child_slot)) - { - HandleChildCrash(pid, exitstatus, namebuf); - return true; - } - - /* Get it out of the BackendList and clear out remaining data */ - dlist_delete(&rw->rw_backend->elem); -#ifdef EXEC_BACKEND - ShmemBackendArrayRemove(rw->rw_backend); -#endif - - /* - * It's possible that this background worker started some OTHER - * background worker and asked to be notified when that worker started - * or stopped. If so, cancel any notifications destined for the - * now-dead backend. - */ - if (rw->rw_backend->bgworker_notify) - BackgroundWorkerStopNotifications(rw->rw_pid); - pfree(rw->rw_backend); - rw->rw_backend = NULL; - rw->rw_pid = 0; - rw->rw_child_slot = 0; - ReportBackgroundWorkerExit(&iter); /* report child death */ - - LogChildExit(EXIT_STATUS_0(exitstatus) ? DEBUG1 : LOG, - namebuf, pid, exitstatus); - - return true; - } - - return false; -} - -/* - * CleanupBackend -- cleanup after terminated backend. - * - * Remove all local state associated with backend. - * - * If you change this, see also CleanupBackgroundWorker. - */ -static void -CleanupBackend(int pid, - int exitstatus) /* child's exit status. */ -{ - dlist_mutable_iter iter; - - LogChildExit(DEBUG2, _("server process"), pid, exitstatus); - - /* - * If a backend dies in an ugly way then we must signal all other backends - * to quickdie. If exit status is zero (normal) or one (FATAL exit), we - * assume everything is all right and proceed to remove the backend from - * the active backend list. - */ - -#ifdef WIN32 - - /* - * On win32, also treat ERROR_WAIT_NO_CHILDREN (128) as nonfatal case, - * since that sometimes happens under load when the process fails to start - * properly (long before it starts using shared memory). Microsoft reports - * it is related to mutex failure: - * http://archives.postgresql.org/pgsql-hackers/2010-09/msg00790.php - */ - if (exitstatus == ERROR_WAIT_NO_CHILDREN) - { - LogChildExit(LOG, _("server process"), pid, exitstatus); - exitstatus = 0; - } -#endif - - if (!EXIT_STATUS_0(exitstatus) && !EXIT_STATUS_1(exitstatus)) - { - HandleChildCrash(pid, exitstatus, _("server process")); - return; - } - - dlist_foreach_modify(iter, &BackendList) - { - Backend *bp = dlist_container(Backend, elem, iter.cur); - - if (bp->pid == pid) - { - if (!bp->dead_end) - { - if (!ReleasePostmasterChildSlot(bp->child_slot)) - { - /* - * Uh-oh, the child failed to clean itself up. Treat as a - * crash after all. - */ - HandleChildCrash(pid, exitstatus, _("server process")); - return; - } -#ifdef EXEC_BACKEND - ShmemBackendArrayRemove(bp); -#endif - } - if (bp->bgworker_notify) - { - /* - * This backend may have been slated to receive SIGUSR1 when - * some background worker started or stopped. Cancel those - * notifications, as we don't want to signal PIDs that are not - * PostgreSQL backends. This gets skipped in the (probably - * very common) case where the backend has never requested any - * such notifications. - */ - BackgroundWorkerStopNotifications(bp->pid); - } - dlist_delete(iter.cur); - pfree(bp); - break; - } - } -} - -/* - * HandleChildCrash -- cleanup after failed backend, bgwriter, checkpointer, - * walwriter, autovacuum, archiver, slot sync worker, or background worker. - * - * The objectives here are to clean up our local state about the child - * process, and to signal all other remaining children to quickdie. - */ -static void -HandleChildCrash(int pid, int exitstatus, const char *procname) -{ - dlist_mutable_iter iter; - slist_iter siter; - Backend *bp; - bool take_action; - - /* - * We only log messages and send signals if this is the first process - * crash and we're not doing an immediate shutdown; otherwise, we're only - * here to update postmaster's idea of live processes. If we have already - * signaled children, nonzero exit status is to be expected, so don't - * clutter log. - */ - take_action = !FatalError && Shutdown != ImmediateShutdown; - - if (take_action) - { - LogChildExit(LOG, procname, pid, exitstatus); - ereport(LOG, - (errmsg("terminating any other active server processes"))); - SetQuitSignalReason(PMQUIT_FOR_CRASH); - } - - /* Process background workers. */ - slist_foreach(siter, &BackgroundWorkerList) - { - RegisteredBgWorker *rw; - - rw = slist_container(RegisteredBgWorker, rw_lnode, siter.cur); - if (rw->rw_pid == 0) - continue; /* not running */ - if (rw->rw_pid == pid) - { - /* - * Found entry for freshly-dead worker, so remove it. - */ - (void) ReleasePostmasterChildSlot(rw->rw_child_slot); - dlist_delete(&rw->rw_backend->elem); -#ifdef EXEC_BACKEND - ShmemBackendArrayRemove(rw->rw_backend); -#endif - pfree(rw->rw_backend); - rw->rw_backend = NULL; - rw->rw_pid = 0; - rw->rw_child_slot = 0; - /* don't reset crashed_at */ - /* don't report child stop, either */ - /* Keep looping so we can signal remaining workers */ - } - else - { - /* - * This worker is still alive. Unless we did so already, tell it - * to commit hara-kiri. - */ - if (take_action) - sigquit_child(rw->rw_pid); - } - } - - /* Process regular backends */ - dlist_foreach_modify(iter, &BackendList) - { - bp = dlist_container(Backend, elem, iter.cur); - - if (bp->pid == pid) - { - /* - * Found entry for freshly-dead backend, so remove it. - */ - if (!bp->dead_end) - { - (void) ReleasePostmasterChildSlot(bp->child_slot); -#ifdef EXEC_BACKEND - ShmemBackendArrayRemove(bp); -#endif - } - dlist_delete(iter.cur); - pfree(bp); - /* Keep looping so we can signal remaining backends */ - } - else - { - /* - * This backend is still alive. Unless we did so already, tell it - * to commit hara-kiri. - * - * We could exclude dead_end children here, but at least when - * sending SIGABRT it seems better to include them. - * - * Background workers were already processed above; ignore them - * here. - */ - if (bp->bkend_type == BACKEND_TYPE_BGWORKER) - continue; - - if (take_action) - sigquit_child(bp->pid); - } - } - - /* Take care of the startup process too */ - if (pid == StartupPID) - { - StartupPID = 0; - /* Caller adjusts StartupStatus, so don't touch it here */ - } - else if (StartupPID != 0 && take_action) - { - sigquit_child(StartupPID); - StartupStatus = STARTUP_SIGNALED; - } - - /* Take care of the bgwriter too */ - if (pid == BgWriterPID) - BgWriterPID = 0; - else if (BgWriterPID != 0 && take_action) - sigquit_child(BgWriterPID); - - /* Take care of the checkpointer too */ - if (pid == CheckpointerPID) - CheckpointerPID = 0; - else if (CheckpointerPID != 0 && take_action) - sigquit_child(CheckpointerPID); - - /* Take care of the walwriter too */ - if (pid == WalWriterPID) - WalWriterPID = 0; - else if (WalWriterPID != 0 && take_action) - sigquit_child(WalWriterPID); - - /* Take care of the walreceiver too */ - if (pid == WalReceiverPID) - WalReceiverPID = 0; - else if (WalReceiverPID != 0 && take_action) - sigquit_child(WalReceiverPID); - - /* Take care of the walsummarizer too */ - if (pid == WalSummarizerPID) - WalSummarizerPID = 0; - else if (WalSummarizerPID != 0 && take_action) - sigquit_child(WalSummarizerPID); - - /* Take care of the autovacuum launcher too */ - if (pid == AutoVacPID) - AutoVacPID = 0; - else if (AutoVacPID != 0 && take_action) - sigquit_child(AutoVacPID); - - /* Take care of the archiver too */ - if (pid == PgArchPID) - PgArchPID = 0; - else if (PgArchPID != 0 && take_action) - sigquit_child(PgArchPID); - - /* Take care of the slot sync worker too */ - if (pid == SlotSyncWorkerPID) - SlotSyncWorkerPID = 0; - else if (SlotSyncWorkerPID != 0 && take_action) - sigquit_child(SlotSyncWorkerPID); - - /* We do NOT restart the syslogger */ - - if (Shutdown != ImmediateShutdown) - FatalError = true; - - /* We now transit into a state of waiting for children to die */ - if (pmState == PM_RECOVERY || - pmState == PM_HOT_STANDBY || - pmState == PM_RUN || - pmState == PM_STOP_BACKENDS || - pmState == PM_SHUTDOWN) - pmState = PM_WAIT_BACKENDS; - - /* - * .. and if this doesn't happen quickly enough, now the clock is ticking - * for us to kill them without mercy. - */ - if (AbortStartTime == 0) - AbortStartTime = time(NULL); -} - -/* - * Log the death of a child process. - */ -static void -LogChildExit(int lev, const char *procname, int pid, int exitstatus) -{ - /* - * size of activity_buffer is arbitrary, but set equal to default - * track_activity_query_size - */ - char activity_buffer[1024]; - const char *activity = NULL; - - if (!EXIT_STATUS_0(exitstatus)) - activity = pgstat_get_crashed_backend_activity(pid, - activity_buffer, - sizeof(activity_buffer)); - - if (WIFEXITED(exitstatus)) - ereport(lev, - - /*------ - translator: %s is a noun phrase describing a child process, such as - "server process" */ - (errmsg("%s (PID %d) exited with exit code %d", - procname, pid, WEXITSTATUS(exitstatus)), - activity ? errdetail("Failed process was running: %s", activity) : 0)); - else if (WIFSIGNALED(exitstatus)) - { -#if defined(WIN32) - ereport(lev, - - /*------ - translator: %s is a noun phrase describing a child process, such as - "server process" */ - (errmsg("%s (PID %d) was terminated by exception 0x%X", - procname, pid, WTERMSIG(exitstatus)), - errhint("See C include file \"ntstatus.h\" for a description of the hexadecimal value."), - activity ? errdetail("Failed process was running: %s", activity) : 0)); -#else - ereport(lev, - - /*------ - translator: %s is a noun phrase describing a child process, such as - "server process" */ - (errmsg("%s (PID %d) was terminated by signal %d: %s", - procname, pid, WTERMSIG(exitstatus), - pg_strsignal(WTERMSIG(exitstatus))), - activity ? errdetail("Failed process was running: %s", activity) : 0)); -#endif - } - else - ereport(lev, - - /*------ - translator: %s is a noun phrase describing a child process, such as - "server process" */ - (errmsg("%s (PID %d) exited with unrecognized status %d", - procname, pid, exitstatus), - activity ? errdetail("Failed process was running: %s", activity) : 0)); -} - -/* - * Advance the postmaster's state machine and take actions as appropriate - * - * This is common code for process_pm_shutdown_request(), - * process_pm_child_exit() and process_pm_pmsignal(), which process the signals - * that might mean we need to change state. - */ -static void -PostmasterStateMachine(void) -{ - /* If we're doing a smart shutdown, try to advance that state. */ - if (pmState == PM_RUN || pmState == PM_HOT_STANDBY) - { - if (!connsAllowed) - { + UpdatePMState(PM_RUN); + connsAllowed = true; + /* - * This state ends when we have no normal client backends running. - * Then we're ready to stop other children. + * At the next iteration of the postmaster's main loop, we will + * crank up the background tasks like the autovacuum launcher and + * background workers that were not started earlier already. */ - if (CountChildren(BACKEND_TYPE_NORMAL) == 0) - pmState = PM_STOP_BACKENDS; + StartWorkerNeeded = true; + + /* at this point we are really open for business */ + ereport(LOG, + (errmsg("database system is ready to accept connections"))); + + /* Report status */ + AddToDataDirLockFile(LOCK_FILE_LINE_PM_STATUS, PM_STATUS_READY); +#ifdef USE_SYSTEMD + sd_notify(0, "READY=1"); +#endif + + continue; } - } - /* - * If we're ready to do so, signal child processes to shut down. (This - * isn't a persistent state, but treating it as a distinct pmState allows - * us to share this code across multiple shutdown code paths.) - */ - if (pmState == PM_STOP_BACKENDS) - { /* - * Forget any pending requests for background workers, since we're no - * longer willing to launch any new workers. (If additional requests - * arrive, BackgroundWorkerStateChange will reject them.) + * Was it the bgwriter? Normal exit can be ignored; we'll start a new + * one at the next iteration of the postmaster's main loop, if + * necessary. Any other exit condition is treated as a crash. */ - ForgetUnstartedBackgroundWorkers(); - - /* Signal all backend children except walsenders */ - SignalSomeChildren(SIGTERM, - BACKEND_TYPE_ALL - BACKEND_TYPE_WALSND); - /* and the autovac launcher too */ - if (AutoVacPID != 0) - signal_child(AutoVacPID, SIGTERM); - /* and the bgwriter too */ - if (BgWriterPID != 0) - signal_child(BgWriterPID, SIGTERM); - /* and the walwriter too */ - if (WalWriterPID != 0) - signal_child(WalWriterPID, SIGTERM); - /* If we're in recovery, also stop startup and walreceiver procs */ - if (StartupPID != 0) - signal_child(StartupPID, SIGTERM); - if (WalReceiverPID != 0) - signal_child(WalReceiverPID, SIGTERM); - if (WalSummarizerPID != 0) - signal_child(WalSummarizerPID, SIGTERM); - if (SlotSyncWorkerPID != 0) - signal_child(SlotSyncWorkerPID, SIGTERM); - /* checkpointer, archiver, stats, and syslogger may continue for now */ - - /* Now transition to PM_WAIT_BACKENDS state to wait for them to die */ - pmState = PM_WAIT_BACKENDS; - } + if (BgWriterPMChild && pid == BgWriterPMChild->pid) + { + ReleasePostmasterChildSlot(BgWriterPMChild); + BgWriterPMChild = NULL; + if (!EXIT_STATUS_0(exitstatus)) + HandleChildCrash(pid, exitstatus, + _("background writer process")); + continue; + } - /* - * If we are in a state-machine state that implies waiting for backends to - * exit, see if they're all gone, and change state if so. - */ - if (pmState == PM_WAIT_BACKENDS) - { /* - * PM_WAIT_BACKENDS state ends when we have no regular backends - * (including autovac workers), no bgworkers (including unconnected - * ones), and no walwriter, autovac launcher, bgwriter or slot sync - * worker. If we are doing crash recovery or an immediate shutdown - * then we expect the checkpointer to exit as well, otherwise not. The - * stats and syslogger processes are disregarded since they are not - * connected to shared memory; we also disregard dead_end children - * here. Walsenders and archiver are also disregarded, they will be - * terminated later after writing the checkpoint record. + * Was it the checkpointer? */ ++<<<<<<< ours + if (CheckpointerPMChild && pid == CheckpointerPMChild->pid) + { + ReleasePostmasterChildSlot(CheckpointerPMChild); + CheckpointerPMChild = NULL; + if (EXIT_STATUS_0(exitstatus) && pmState == PM_WAIT_CHECKPOINTER) ++======= + if (CountChildren(BACKEND_TYPE_ALL - BACKEND_TYPE_WALSND) == 0 && + StartupPID == 0 && + WalReceiverPID == 0 && + WalSummarizerPID == 0 && + BgWriterPID == 0 && + (CheckpointerPID == 0 || + (!FatalError && Shutdown < ImmediateShutdown) || + (FatalError && CheckpointerPID != 0)) && + WalWriterPID == 0 && + AutoVacPID == 0 && + SlotSyncWorkerPID == 0) + { + if (CheckpointerPID == 0 && + (Shutdown >= ImmediateShutdown || FatalError)) ++>>>>>>> theirs { /* - * Start waiting for dead_end children to die. This state - * change causes ServerLoop to stop creating new ones. - */ - pmState = PM_WAIT_DEAD_END; - - /* - * We already SIGQUIT'd the archiver and stats processes, if - * any, when we started immediate shutdown or entered - * FatalError state. + * OK, we saw normal exit of the checkpointer after it's been + * told to shut down. We know checkpointer wrote a shutdown + * checkpoint, otherwise we'd still be in + * PM_WAIT_XLOG_SHUTDOWN state. + * + * At this point only dead-end children and logger should be + * left. */ + UpdatePMState(PM_WAIT_DEAD_END); + ConfigurePostmasterWaitSet(false); + SignalChildren(SIGTERM, btmask_all_except(B_LOGGER)); } - else + else if (Shutdown > NoShutdown && Shutdown < ImmediateShutdown) { /* - * If we get here, we are proceeding with normal shutdown. All - * the regular children are gone, and it's time to tell the - * checkpointer to do a shutdown checkpoint. + * Any unexpected exit of the checkpointer (including FATAL + * exit) is treated as a crash. */ - Assert(Shutdown > NoShutdown); - /* Start the checkpointer if not running */ - if (CheckpointerPID == 0) - CheckpointerPID = StartChildProcess(CheckpointerProcess); - /* And tell it to shut down */ - if (CheckpointerPID != 0) - { - signal_child(CheckpointerPID, SIGUSR2); - pmState = PM_SHUTDOWN; - } - else - { - /* - * If we failed to fork a checkpointer, just shut down. - * Any required cleanup will happen at next restart. We - * set FatalError so that an "abnormal shutdown" message - * gets logged when we exit. - * - * We don't consult send_abort_for_crash here, as it's - * unlikely that dumping cores would illuminate the reason - * for checkpointer fork failure. - */ - FatalError = true; - pmState = PM_WAIT_DEAD_END; - - /* Kill the walsenders and archiver too */ - SignalChildren(SIGQUIT); - if (PgArchPID != 0) - signal_child(PgArchPID, SIGQUIT); - } + HandleChildCrash(pid, exitstatus, + _("checkpointer process")); } ++<<<<<<< ours + + continue; ++======= + else + { + /* + * Either it's an immediate shutdown or a child crashed, and + * we're still waiting for all the children to quit. The + * checkpointer was already told to quit. + */ + Assert(Shutdown == ImmediateShutdown || + (Shutdown == NoShutdown && FatalError)); + } ++>>>>>>> theirs } - } - if (pmState == PM_SHUTDOWN_2) - { /* - * PM_SHUTDOWN_2 state ends when there's no other children than - * dead_end children left. There shouldn't be any regular backends - * left by now anyway; what we're really waiting for is walsenders and - * archiver. + * Was it the wal writer? Normal exit can be ignored; we'll start a + * new one at the next iteration of the postmaster's main loop, if + * necessary. Any other exit condition is treated as a crash. */ - if (PgArchPID == 0 && CountChildren(BACKEND_TYPE_ALL) == 0) + if (WalWriterPMChild && pid == WalWriterPMChild->pid) { - pmState = PM_WAIT_DEAD_END; + ReleasePostmasterChildSlot(WalWriterPMChild); + WalWriterPMChild = NULL; + if (!EXIT_STATUS_0(exitstatus)) + HandleChildCrash(pid, exitstatus, + _("WAL writer process")); + continue; } - } - - if (pmState == PM_WAIT_DEAD_END) - { - /* Don't allow any new socket connection events. */ - ConfigurePostmasterWaitSet(false); /* - * PM_WAIT_DEAD_END state ends when the BackendList is entirely empty - * (ie, no dead_end children remain), and the archiver is gone too. - * - * The reason we wait for those two is to protect them against a new - * postmaster starting conflicting subprocesses; this isn't an - * ironclad protection, but it at least helps in the - * shutdown-and-immediately-restart scenario. Note that they have - * already been sent appropriate shutdown signals, either during a - * normal state transition leading up to PM_WAIT_DEAD_END, or during - * FatalError processing. + * Was it the wal receiver? If exit status is zero (normal) or one + * (FATAL exit), we assume everything is all right just like normal + * backends. (If we need a new wal receiver, we'll start one at the + * next iteration of the postmaster's main loop.) */ - if (dlist_is_empty(&BackendList) && PgArchPID == 0) + if (WalReceiverPMChild && pid == WalReceiverPMChild->pid) { - /* These other guys should be dead already */ - Assert(StartupPID == 0); - Assert(WalReceiverPID == 0); - Assert(WalSummarizerPID == 0); - Assert(BgWriterPID == 0); - Assert(CheckpointerPID == 0); - Assert(WalWriterPID == 0); - Assert(AutoVacPID == 0); - Assert(SlotSyncWorkerPID == 0); - /* syslogger is not considered here */ - pmState = PM_NO_CHILDREN; + ReleasePostmasterChildSlot(WalReceiverPMChild); + WalReceiverPMChild = NULL; + if (!EXIT_STATUS_0(exitstatus) && !EXIT_STATUS_1(exitstatus)) + HandleChildCrash(pid, exitstatus, + _("WAL receiver process")); + continue; } - } - /* - * If we've been told to shut down, we exit as soon as there are no - * remaining children. If there was a crash, cleanup will occur at the - * next startup. (Before PostgreSQL 8.3, we tried to recover from the - * crash before exiting, but that seems unwise if we are quitting because - * we got SIGTERM from init --- there may well not be time for recovery - * before init decides to SIGKILL us.) - * - * Note that the syslogger continues to run. It will exit when it sees - * EOF on its input pipe, which happens when there are no more upstream - * processes. - */ - if (Shutdown > NoShutdown && pmState == PM_NO_CHILDREN) - { - if (FatalError) - { - ereport(LOG, (errmsg("abnormal database system shutdown"))); - ExitPostmaster(1); - } - else + /* + * Was it the wal summarizer? Normal exit can be ignored; we'll start + * a new one at the next iteration of the postmaster's main loop, if + * necessary. Any other exit condition is treated as a crash. + */ + if (WalSummarizerPMChild && pid == WalSummarizerPMChild->pid) { - /* - * Normal exit from the postmaster is here. We don't need to log - * anything here, since the UnlinkLockFiles proc_exit callback - * will do so, and that should be the last user-visible action. - */ - ExitPostmaster(0); + ReleasePostmasterChildSlot(WalSummarizerPMChild); + WalSummarizerPMChild = NULL; + if (!EXIT_STATUS_0(exitstatus)) + HandleChildCrash(pid, exitstatus, + _("WAL summarizer process")); + continue; } - } - /* - * If the startup process failed, or the user does not want an automatic - * restart after backend crashes, wait for all non-syslogger children to - * exit, and then exit postmaster. We don't try to reinitialize when the - * startup process fails, because more than likely it will just fail again - * and we will keep trying forever. - */ - if (pmState == PM_NO_CHILDREN) - { - if (StartupStatus == STARTUP_CRASHED) + /* + * Was it the autovacuum launcher? Normal exit can be ignored; we'll + * start a new one at the next iteration of the postmaster's main + * loop, if necessary. Any other exit condition is treated as a + * crash. + */ + if (AutoVacLauncherPMChild && pid == AutoVacLauncherPMChild->pid) { - ereport(LOG, - (errmsg("shutting down due to startup process failure"))); - ExitPostmaster(1); + ReleasePostmasterChildSlot(AutoVacLauncherPMChild); + AutoVacLauncherPMChild = NULL; + if (!EXIT_STATUS_0(exitstatus)) + HandleChildCrash(pid, exitstatus, + _("autovacuum launcher process")); + continue; } - if (!restart_after_crash) + + /* + * Was it the archiver? If exit status is zero (normal) or one (FATAL + * exit), we assume everything is all right just like normal backends + * and just try to start a new one on the next cycle of the + * postmaster's main loop, to retry archiving remaining files. + */ + if (PgArchPMChild && pid == PgArchPMChild->pid) { - ereport(LOG, - (errmsg("shutting down because restart_after_crash is off"))); - ExitPostmaster(1); + ReleasePostmasterChildSlot(PgArchPMChild); + PgArchPMChild = NULL; + if (!EXIT_STATUS_0(exitstatus) && !EXIT_STATUS_1(exitstatus)) + HandleChildCrash(pid, exitstatus, + _("archiver process")); + continue; } - } - - /* - * If we need to recover from a crash, wait for all non-syslogger children - * to exit, then reset shmem and start the startup process. - */ - if (FatalError && pmState == PM_NO_CHILDREN) - { - ereport(LOG, - (errmsg("all server processes terminated; reinitializing"))); - - /* remove leftover temporary files after a crash */ - if (remove_temp_files_after_crash) - RemovePgTempFiles(); - - /* allow background workers to immediately restart */ - ResetBackgroundWorkerCrashTimes(); - - shmem_exit(1); - - /* re-read control file into local memory */ - LocalProcessControlFile(true); - - /* re-create shared memory and semaphores */ - CreateSharedMemoryAndSemaphores(); - - StartupPID = StartChildProcess(StartupProcess); - Assert(StartupPID != 0); - StartupStatus = STARTUP_RUNNING; - pmState = PM_STARTUP; - /* crash recovery started, reset SIGKILL flag */ - AbortStartTime = 0; - - /* start accepting server socket connection events again */ - ConfigurePostmasterWaitSet(true); - } -} - -/* - * Send a signal to a postmaster child process - * - * On systems that have setsid(), each child process sets itself up as a - * process group leader. For signals that are generally interpreted in the - * appropriate fashion, we signal the entire process group not just the - * direct child process. This allows us to, for example, SIGQUIT a blocked - * archive_recovery script, or SIGINT a script being run by a backend via - * system(). - * - * There is a race condition for recently-forked children: they might not - * have executed setsid() yet. So we signal the child directly as well as - * the group. We assume such a child will handle the signal before trying - * to spawn any grandchild processes. We also assume that signaling the - * child twice will not cause any problems. - */ -static void -signal_child(pid_t pid, int signal) -{ - if (kill(pid, signal) < 0) - elog(DEBUG3, "kill(%ld,%d) failed: %m", (long) pid, signal); -#ifdef HAVE_SETSID - switch (signal) - { - case SIGINT: - case SIGTERM: - case SIGQUIT: - case SIGKILL: - case SIGABRT: - if (kill(-pid, signal) < 0) - elog(DEBUG3, "kill(%ld,%d) failed: %m", (long) (-pid), signal); - break; - default: - break; - } -#endif -} + /* Was it the system logger? If so, try to start a new one */ + if (SysLoggerPMChild && pid == SysLoggerPMChild->pid) + { + ReleasePostmasterChildSlot(SysLoggerPMChild); + SysLoggerPMChild = NULL; -/* - * Convenience function for killing a child process after a crash of some - * other child process. We log the action at a higher level than we would - * otherwise do, and we apply send_abort_for_crash to decide which signal - * to send. Normally it's SIGQUIT -- and most other comments in this file - * are written on the assumption that it is -- but developers might prefer - * to use SIGABRT to collect per-child core dumps. - */ -static void -sigquit_child(pid_t pid) -{ - ereport(DEBUG2, - (errmsg_internal("sending %s to process %d", - (send_abort_for_crash ? "SIGABRT" : "SIGQUIT"), - (int) pid))); - signal_child(pid, (send_abort_for_crash ? SIGABRT : SIGQUIT)); -} + /* for safety's sake, launch new logger *first* */ + if (Logging_collector) + StartSysLogger(); -/* - * Send a signal to the targeted children (but NOT special children; - * dead_end children are never signaled, either). - */ -static bool -SignalSomeChildren(int signal, int target) -{ - dlist_iter iter; - bool signaled = false; + if (!EXIT_STATUS_0(exitstatus)) + LogChildExit(LOG, _("system logger process"), + pid, exitstatus); + continue; + } - dlist_foreach(iter, &BackendList) - { - Backend *bp = dlist_container(Backend, elem, iter.cur); + /* + * Was it the slot sync worker? Normal exit or FATAL exit can be + * ignored (FATAL can be caused by libpqwalreceiver on receiving + * shutdown request by the startup process during promotion); we'll + * start a new one at the next iteration of the postmaster's main + * loop, if necessary. Any other exit condition is treated as a crash. + */ + if (SlotSyncWorkerPMChild && pid == SlotSyncWorkerPMChild->pid) + { + ReleasePostmasterChildSlot(SlotSyncWorkerPMChild); + SlotSyncWorkerPMChild = NULL; + if (!EXIT_STATUS_0(exitstatus) && !EXIT_STATUS_1(exitstatus)) + HandleChildCrash(pid, exitstatus, + _("slot sync worker process")); + continue; + } + + /* Was it an IO worker? */ + if (maybe_reap_io_worker(pid)) + { + if (!EXIT_STATUS_0(exitstatus) && !EXIT_STATUS_1(exitstatus)) + HandleChildCrash(pid, exitstatus, _("io worker")); - if (bp->dead_end) + maybe_adjust_io_workers(); continue; + } /* - * Since target == BACKEND_TYPE_ALL is the most common case, we test - * it first and avoid touching shared memory for every child. + * Was it a backend or a background worker? */ - if (target != BACKEND_TYPE_ALL) + pmchild = FindPostmasterChildByPid(pid); + if (pmchild) { - /* - * Assign bkend_type for any recently announced WAL Sender - * processes. - */ - if (bp->bkend_type == BACKEND_TYPE_NORMAL && - IsPostmasterChildWalSender(bp->child_slot)) - bp->bkend_type = BACKEND_TYPE_WALSND; + CleanupBackend(pmchild, exitstatus); + } - if (!(target & bp->bkend_type)) - continue; + /* + * We don't know anything about this child process. That's highly + * unexpected, as we do track all the child processes that we fork. + */ + else + { + if (!EXIT_STATUS_0(exitstatus) && !EXIT_STATUS_1(exitstatus)) + HandleChildCrash(pid, exitstatus, _("untracked child process")); + else + LogChildExit(LOG, _("untracked child process"), pid, exitstatus); } + } /* loop over pending child-death reports */ - ereport(DEBUG4, - (errmsg_internal("sending signal %d to process %d", - signal, (int) bp->pid))); - signal_child(bp->pid, signal); - signaled = true; - } - return signaled; + /* + * After cleaning out the SIGCHLD queue, see if we have any state changes + * or actions to make. + */ + PostmasterStateMachine(); } /*