Provide much better wait information in pg_stat_activity.

When a process is waiting for a heavyweight lock, we will now indicate
the type of heavyweight lock for which it is waiting.  Also, you can
now see when a process is waiting for a lightweight lock - in which
case we will indicate the individual lock name or the tranche, as
appropriate - or for a buffer pin.

Amit Kapila, Ildus Kurbangaliev, reviewed by me.  Lots of helpful
discussion and suggestions by many others, including Alexander
Korotkov, Vladimir Borodin, and many others.

Author: robertmhaas

doc/src/sgml/monitoring.sgml

@@ -2447,13 +2447,14 @@ AbortTransaction(void)
 	 */
 	LWLockReleaseAll();
 
+	/* Clear wait information and command progress indicator */
+	pgstat_report_wait_end();
+	pgstat_progress_end_command();
+
 	/* Clean up buffer I/O and buffer context locks, too */
 	AbortBufferIO();
 	UnlockBuffers();
 
-	/* Clear command progress indicator */
-	pgstat_progress_end_command();
-
 	/* Reset WAL record construction state */
 	XLogResetInsertion();
 
@@ -4541,9 +4542,10 @@ AbortSubTransaction(void)
 	 */
 	LWLockReleaseAll();
 
+	pgstat_report_wait_end();
+	pgstat_progress_end_command();
 	AbortBufferIO();
 	UnlockBuffers();
-	pgstat_progress_end_command();
 
 	/* Reset WAL record construction state */
 	XLogResetInsertion();
@@ -4653,6 +4655,9 @@ AbortSubTransaction(void)
 	 */
 	XactReadOnly = s->prevXactReadOnly;
 
+	/* Report wait end here, when there is no further possibility of wait */
+	pgstat_report_wait_end();
+
 	RESUME_INTERRUPTS();
 }
 

src/backend/access/transam/xact.c

@@ -26,6 +26,7 @@
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
 #include "pg_getopt.h"
+#include "pgstat.h"
 #include "postmaster/bgwriter.h"
 #include "postmaster/startup.h"
 #include "postmaster/walwriter.h"
@@ -534,6 +535,7 @@ static void
 ShutdownAuxiliaryProcess(int code, Datum arg)
 {
 	LWLockReleaseAll();
+	pgstat_report_wait_end();
 }
 
 /* ----------------------------------------------------------------

src/backend/bootstrap/bootstrap.c

@@ -636,7 +636,8 @@ CREATE VIEW pg_stat_activity AS
             S.xact_start,
             S.query_start,
             S.state_change,
-            S.waiting,
+            S.wait_event_type,
+            S.wait_event,
             S.state,
             S.backend_xid,
             s.backend_xmin,

src/backend/catalog/system_views.sql

@@ -224,6 +224,9 @@ BackgroundWriterMain(void)
 		 * It's not clear we need it elsewhere, but shouldn't hurt.
 		 */
 		smgrcloseall();
+
+		/* Report wait end here, when there is no further possibility of wait */
+		pgstat_report_wait_end();
 	}
 
 	/* We can now handle ereport(ERROR) */

src/backend/postmaster/bgwriter.c

@@ -273,6 +273,7 @@ CheckpointerMain(void)
 		 * files.
 		 */
 		LWLockReleaseAll();
+		pgstat_report_wait_end();
 		AbortBufferIO();
 		UnlockBuffers();
 		/* buffer pins are released here: */

src/backend/postmaster/checkpointer.c

@@ -48,12 +48,12 @@
 #include "postmaster/autovacuum.h"
 #include "postmaster/fork_process.h"
 #include "postmaster/postmaster.h"
-#include "storage/proc.h"
 #include "storage/backendid.h"
 #include "storage/dsm.h"
 #include "storage/fd.h"
 #include "storage/ipc.h"
 #include "storage/latch.h"
+#include "storage/lmgr.h"
 #include "storage/pg_shmem.h"
 #include "storage/procsignal.h"
 #include "storage/sinvaladt.h"
@@ -2723,7 +2723,6 @@ pgstat_bestart(void)
 #else
 	beentry->st_ssl = false;
 #endif
-	beentry->st_waiting = false;
 	beentry->st_state = STATE_UNDEFINED;
 	beentry->st_appname[0] = '\0';
 	beentry->st_activity[0] = '\0';
@@ -2810,6 +2809,8 @@ pgstat_report_activity(BackendState state, const char *cmd_str)
 	{
 		if (beentry->st_state != STATE_DISABLED)
 		{
+			volatile PGPROC *proc = MyProc;
+
 			/*
 			 * track_activities is disabled, but we last reported a
 			 * non-disabled state.  As our final update, change the state and
@@ -2820,9 +2821,9 @@ pgstat_report_activity(BackendState state, const char *cmd_str)
 			beentry->st_state_start_timestamp = 0;
 			beentry->st_activity[0] = '\0';
 			beentry->st_activity_start_timestamp = 0;
-			/* st_xact_start_timestamp and st_waiting are also disabled */
+			/* st_xact_start_timestamp and wait_event_info are also disabled */
 			beentry->st_xact_start_timestamp = 0;
-			beentry->st_waiting = false;
+			proc->wait_event_info = 0;
 			pgstat_increment_changecount_after(beentry);
 		}
 		return;
@@ -2978,32 +2979,6 @@ pgstat_report_xact_timestamp(TimestampTz tstamp)
 	pgstat_increment_changecount_after(beentry);
 }
 
-/* ----------
- * pgstat_report_waiting() -
- *
- *	Called from lock manager to report beginning or end of a lock wait.
- *
- * NB: this *must* be able to survive being called before MyBEEntry has been
- * initialized.
- * ----------
- */
-void
-pgstat_report_waiting(bool waiting)
-{
-	volatile PgBackendStatus *beentry = MyBEEntry;
-
-	if (!pgstat_track_activities || !beentry)
-		return;
-
-	/*
-	 * Since this is a single-byte field in a struct that only this process
-	 * may modify, there seems no need to bother with the st_changecount
-	 * protocol.  The update must appear atomic in any case.
-	 */
-	beentry->st_waiting = waiting;
-}
-
-
 /* ----------
  * pgstat_read_current_status() -
  *
@@ -3119,6 +3094,87 @@ pgstat_read_current_status(void)
 	localBackendStatusTable = localtable;
 }
 
+/* ----------
+ * pgstat_get_wait_event_type() -
+ *
+ *	Return a string representing the current wait event type, backend is
+ *	waiting on.
+ */
+const char *
+pgstat_get_wait_event_type(uint32 wait_event_info)
+{
+	uint8		classId;
+	const char *event_type;
+
+	/* report process as not waiting. */
+	if (wait_event_info == 0)
+		return NULL;
+
+	wait_event_info = wait_event_info >> 24;
+	classId = wait_event_info & 0XFF;
+
+	switch (classId)
+	{
+		case WAIT_LWLOCK_NAMED:
+			event_type = "LWLockNamed";
+			break;
+		case WAIT_LWLOCK_TRANCHE:
+			event_type = "LWLockTranche";
+			break;
+		case WAIT_LOCK:
+			event_type = "Lock";
+			break;
+		case WAIT_BUFFER_PIN:
+			event_type = "BufferPin";
+			break;
+		default:
+			event_type = "???";
+			break;
+	}
+
+	return event_type;
+}
+
+/* ----------
+ * pgstat_get_wait_event() -
+ *
+ *	Return a string representing the current wait event, backend is
+ *	waiting on.
+ */
+const char *
+pgstat_get_wait_event(uint32 wait_event_info)
+{
+	uint8		classId;
+	uint16		eventId;
+	const char *event_name;
+
+	/* report process as not waiting. */
+	if (wait_event_info == 0)
+		return NULL;
+
+	eventId = wait_event_info & ((1 << 24) - 1);
+	wait_event_info = wait_event_info >> 24;
+	classId = wait_event_info & 0XFF;
+
+	switch (classId)
+	{
+		case WAIT_LWLOCK_NAMED:
+		case WAIT_LWLOCK_TRANCHE:
+			event_name = GetLWLockIdentifier(classId, eventId);
+			break;
+		case WAIT_LOCK:
+			event_name = GetLockNameFromTagType(eventId);
+			break;
+		case WAIT_BUFFER_PIN:
+			event_name = "BufferPin";
+			break;
+		default:
+			event_name = "unknown wait event";
+			break;
+	}
+
+	return event_name;
+}
 
 /* ----------
  * pgstat_get_backend_current_activity() -

src/backend/postmaster/pgstat.c

@@ -47,6 +47,7 @@
 #include "access/xlog.h"
 #include "libpq/pqsignal.h"
 #include "miscadmin.h"
+#include "pgstat.h"
 #include "postmaster/walwriter.h"
 #include "storage/bufmgr.h"
 #include "storage/fd.h"
@@ -168,6 +169,7 @@ WalWriterMain(void)
 		 * about in walwriter, but we do have LWLocks, and perhaps buffers?
 		 */
 		LWLockReleaseAll();
+		pgstat_report_wait_end();
 		AbortBufferIO();
 		UnlockBuffers();
 		/* buffer pins are released here: */

src/backend/postmaster/walwriter.c

@@ -55,6 +55,7 @@
 #include "libpq/pqformat.h"
 #include "miscadmin.h"
 #include "nodes/replnodes.h"
+#include "pgstat.h"
 #include "replication/basebackup.h"
 #include "replication/decode.h"
 #include "replication/logical.h"
@@ -252,6 +253,7 @@ void
 WalSndErrorCleanup(void)
 {
 	LWLockReleaseAll();
+	pgstat_report_wait_end();
 
 	if (sendFile >= 0)
 	{

src/backend/replication/walsender.c

@@ -3351,6 +3351,9 @@ LockBufferForCleanup(Buffer buffer)
 		UnlockBufHdr(bufHdr);
 		LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
 
+		/* Report the wait */
+		pgstat_report_wait_start(WAIT_BUFFER_PIN, 0);
+
 		/* Wait to be signaled by UnpinBuffer() */
 		if (InHotStandby)
 		{
@@ -3364,6 +3367,8 @@ LockBufferForCleanup(Buffer buffer)
 		else
 			ProcWaitForSignal();
 
+		pgstat_report_wait_end();
+
 		/*
 		 * Remove flag marking us as waiter. Normally this will not be set
 		 * anymore, but ProcWaitForSignal() can return for other signals as

src/backend/storage/buffer/bufmgr.c

@@ -994,3 +994,26 @@ DescribeLockTag(StringInfo buf, const LOCKTAG *tag)
 			break;
 	}
 }
+
+/*
+ * GetLockNameFromTagType
+ *
+ *	Given locktag type, return the corresponding lock name.
+ */
+const char *
+GetLockNameFromTagType(uint16 locktag_type)
+{
+	const char *locktypename;
+	char		tnbuf[32];
+
+	if (locktag_type <= LOCKTAG_LAST_TYPE)
+		locktypename = LockTagTypeNames[locktag_type];
+	else
+	{
+		snprintf(tnbuf, sizeof(tnbuf), "unknown %d",
+				 (int) locktag_type);
+		locktypename = tnbuf;
+	}
+
+	return locktypename;
+}

src/backend/storage/lmgr/lmgr.c

@@ -1676,7 +1676,7 @@ WaitOnLock(LOCALLOCK *locallock, ResourceOwner owner)
 		set_ps_display(new_status, false);
 		new_status[len] = '\0'; /* truncate off " waiting" */
 	}
-	pgstat_report_waiting(true);
+	pgstat_report_wait_start(WAIT_LOCK, locallock->tag.lock.locktag_type);
 
 	awaitedLock = locallock;
 	awaitedOwner = owner;
@@ -1724,7 +1724,7 @@ WaitOnLock(LOCALLOCK *locallock, ResourceOwner owner)
 		/* In this path, awaitedLock remains set until LockErrorCleanup */
 
 		/* Report change to non-waiting status */
-		pgstat_report_waiting(false);
+		pgstat_report_wait_end();
 		if (update_process_title)
 		{
 			set_ps_display(new_status, false);
@@ -1739,7 +1739,7 @@ WaitOnLock(LOCALLOCK *locallock, ResourceOwner owner)
 	awaitedLock = NULL;
 
 	/* Report change to non-waiting status */
-	pgstat_report_waiting(false);
+	pgstat_report_wait_end();
 	if (update_process_title)
 	{
 		set_ps_display(new_status, false);