Port -DEXTRA_DYNAMIC_MEMORY_DEBUG to GPDB 7

This commit is a rework of the same macro from GPDB 6 (commit 1ddd6b0, #403)

The patch enhances the output of MemoryContextStats() by including a list of
memory allocations along with their locations for each memory context. To enable
the feature, add -DEXTRA_DYNAMIC_MEMORY_DEBUG to CFLAGS during ./configure. At
runtime, invoke MemoryContextStats(TopMemoryContext) or any other context as
needed. The output will display the top allocations for each context following
the default memory statistics.

Co-authored-by: Evgeniy Ratkov <[email protected]>

src/backend/utils/mmgr/aset.c

@@ -52,6 +52,8 @@
 #include "utils/memutils.h"
 #include "utils/gp_alloc.h"
 
+#include "utils/hsearch.h"
+
 #include "miscadmin.h"
 
 /* Define this to detail debug alloc information */
@@ -180,6 +182,10 @@ typedef struct AllocSetContext
 
 	Size		currentAllocated;
 	Size		peakAllocated;
+
+#ifdef EXTRA_DYNAMIC_MEMORY_DEBUG
+	HTAB *chunkTable;
+#endif
 } AllocSetContext;
 
 typedef AllocSetContext *AllocSet;
@@ -266,6 +272,20 @@ typedef struct AllocChunkData
 #define ALLOCCHUNK_RAWSIZE  (SIZEOF_SIZE_T + SIZEOF_VOID_P)
 #endif							/* MEMORY_CONTEXT_CHECKING */
 
+#ifdef EXTRA_DYNAMIC_MEMORY_DEBUG
+	/*
+	 * extra information about callee, file, and line number of each
+	 * allocation
+	 */
+	MemoryContextChunkInfo info;
+
+#define ALLOCCHUNK_RAWSIZE_EXTRA ALLOCCHUNK_RAWSIZE
+#undef ALLOCCHUNK_RAWSIZE
+
+#define ALLOCCHUNK_RAWSIZE \
+	(ALLOCCHUNK_RAWSIZE_EXTRA + MEMORYCONTEXTCHUNKINFO_RAWSIZE)
+#endif
+
 	/* ensure proper alignment by adding padding if needed */
 #if (ALLOCCHUNK_RAWSIZE % MAXIMUM_ALIGNOF) != 0
 	char		padding[MAXIMUM_ALIGNOF - ALLOCCHUNK_RAWSIZE % MAXIMUM_ALIGNOF];
@@ -275,6 +295,7 @@ typedef struct AllocChunkData
 	void	   *aset;
 	/* there must not be any padding to reach a MAXALIGN boundary here! */
 
+	/* FIXME: these are obsolete */
 #ifdef CDB_PALLOC_TAGS
 	const char  *alloc_tag;
 	int 		alloc_n;
@@ -425,6 +446,26 @@ static const unsigned char LogTable256[256] =
 #define AllocAllocInfo(_cxt, _chunk)
 #endif
 
+#ifdef EXTRA_DYNAMIC_MEMORY_DEBUG
+HTAB *
+AllocSetTakeChunkTable(MemoryContext context)
+{
+	Assert(AllocSetIsValid(context));
+	Assert(IsA(context, AllocSetContext));
+
+	HTAB *table = ((AllocSet) context)->chunkTable;
+	((AllocSet) context)->chunkTable = NULL;
+
+	return table;
+}
+
+MemoryContextChunkInfo *
+AllocPointerGetChunkInfo(void *ptr)
+{
+	return &AllocPointerGetChunk(ptr)->info;
+}
+#endif
+
 /* ----------
  * AllocSetFreeIndex -
  *
@@ -669,6 +710,10 @@ AllocSetContextCreateInternal(MemoryContext parent,
 	set->currentAllocated = 0;
 	set->peakAllocated = 0;
 
+#if defined EXTRA_DYNAMIC_MEMORY_DEBUG
+	set->chunkTable = NULL;
+#endif
+
 	((MemoryContext) set)->mem_allocated = firstBlockSize;
 
 	return (MemoryContext) set;
@@ -1571,6 +1616,10 @@ AllocSetIsEmpty(MemoryContext context)
 	return false;
 }
 
+#ifdef EXTRA_DYNAMIC_MEMORY_DEBUG
+#include "aset_memory_debug.c"
+#endif
+
 /*
  * AllocSetStats
  *		Compute stats about memory consumption of an allocset.
@@ -1632,6 +1681,10 @@ AllocSetStats(MemoryContext context,
 		totals->totalspace += totalspace;
 		totals->freespace += freespace;
 	}
+
+#ifdef EXTRA_DYNAMIC_MEMORY_DEBUG
+	AllocSetUpdateAllocatedChunkStats(set);
+#endif
 }
 
 static void

src/backend/utils/mmgr/aset_memory_debug.c

@@ -0,0 +1,109 @@
+/* This file should be included into aset.c. */
+
+#include "postgres.h"
+
+#include "utils/palloc_memory_debug.h"
+#include "access/hash.h"
+#include "utils/hsearch.h"
+
+/* Update individual chunk stats. */
+static void
+AllocChunkUpdateStats(HTAB *chunk_table, AllocChunk chunk)
+{
+	bool found = false;
+	MemoryContextChunkTableEntry *r = NULL;
+
+	r = hash_search(chunk_table, &chunk->info.key, HASH_ENTER, &found);
+	Assert(r != NULL);
+
+	if (found)
+	{
+		r->stat.count++;
+		r->stat.bytes += chunk->size;
+	}
+	else
+	{
+		r->info = chunk->info;
+		r->stat.bytes = chunk->size;
+		r->stat.count = 1;
+	}
+}
+
+static uint32
+MemoryContextChunkStatKeyHash(const void *key, Size keysize)
+{
+	Assert(keysize == sizeof(MemoryContextChunkStatKey));
+	return DatumGetUInt32(hash_any((const unsigned char *) key,
+								   keysize));
+}
+
+static int
+MemoryContextChunkStatKeyCompare(const void *a, const void *b,
+								 Size keysize)
+{
+	Assert(keysize == sizeof(MemoryContextChunkStatKey));
+
+	MemoryContextChunkStatKey *lhs = (MemoryContextChunkStatKey *) a;
+	MemoryContextChunkStatKey *rhs = (MemoryContextChunkStatKey *) b;
+
+	return !(strcmp(lhs->parent_func, rhs->parent_func) == 0 &&
+			 lhs->line == rhs->line);
+}
+
+static bool
+AllocSetChunkIsFree(AllocChunk chunk, AllocSet set)
+{
+	if (chunk->size > set->allocChunkLimit)
+		return false;
+
+	AllocChunk free_chunk = set->freelist[AllocSetFreeIndex(chunk->size)];
+
+	while (free_chunk != NULL)
+	{
+		if (free_chunk == chunk)
+			return true;
+
+		free_chunk = (AllocChunk) free_chunk->aset;
+	}
+
+	return false;
+}
+
+/* Update every chunk's stats table inside an AllocSet. */
+static void
+AllocSetUpdateAllocatedChunkStats(AllocSet set)
+{
+	if (set->chunkTable == NULL)
+	{
+		HASHCTL hash_ctl =
+		{
+			.keysize = sizeof(MemoryContextChunkStatKey),
+			.entrysize = sizeof(MemoryContextChunkTableEntry),
+			.hash = MemoryContextChunkStatKeyHash,
+			.match = MemoryContextChunkStatKeyCompare,
+		};
+
+		set->chunkTable = hash_create("AllocSetUpdateAllocatedChunkStats",
+									  DYN_MEM_HTABLE_SIZE, &hash_ctl,
+									  HASH_FUNCTION | HASH_ELEM | HASH_COMPARE);
+	}
+
+	for (AllocBlock block = set->blocks; block != NULL; block = block->next)
+	{
+		AllocChunk chunk;
+
+		for (chunk = (AllocChunk) (((char *) block) + ALLOC_BLOCKHDRSZ);
+			 (char *) chunk < (char *) block->freeptr;
+			 chunk = (AllocChunk) ((char *) chunk + chunk->size + ALLOC_CHUNKHDRSZ))
+		{
+			if (AllocSetChunkIsFree(chunk, set) ||
+				chunk->info.init != EXTRA_DYNAMIC_MEMORY_DEBUG_INIT_MAGIC)
+				continue;
+			if (!chunk->info.key.parent_func || !chunk->info.key.line ||
+				!chunk->info.filename || !chunk->info.func)
+				continue;
+
+			AllocChunkUpdateStats(set->chunkTable, chunk);
+		}
+	}
+}

src/backend/utils/mmgr/mcxt.c

@@ -45,6 +45,10 @@
 #error "If CDB_PALLOC_TAGS is defined, CDB_PALLOC_CALLER_ID must be defined too"
 #endif
 
+#ifdef EXTRA_DYNAMIC_MEMORY_DEBUG
+#include "utils/palloc_memory_debug_undef.h"
+#endif
+
 /*****************************************************************************
  *	  GLOBAL MEMORY															 *
  *****************************************************************************/
@@ -714,6 +718,10 @@ MemoryContextStatsDetail(MemoryContext context, int max_children,
 								 grand_totals.totalspace - grand_totals.freespace)));
 }
 
+#ifdef EXTRA_DYNAMIC_MEMORY_DEBUG
+#include "mcxt_memory_debug.c"
+#endif
+
 /*
  * MemoryContextStatsInternal
  *		One recursion level for MemoryContextStats
@@ -739,6 +747,10 @@ MemoryContextStatsInternal(MemoryContext context, int level,
 							(void *) &level,
 							totals, print_to_stderr);
 
+#ifdef EXTRA_DYNAMIC_MEMORY_DEBUG
+	MemoryContextDumpChunkStats(context, max_children, print_to_stderr);
+#endif
+
 	/*
 	 * Examine children.  If there are more than max_children of them, we do
 	 * not print the rest explicitly, but just summarize them.