Optimize GenerationAlloc() and SlabAlloc()

In a similar effort to 413c18401, separate out the hot and cold paths in
GenerationAlloc() and SlabAlloc() to avoid having to setup the stack frame
for the hot path.

This additionally adjusts how we use the GenerationContext's freeblock.
Freeblock, when set, is now always empty and we only switch to using it
when the current allocation request finds the current block does not have
enough space and the freeblock is large enough to accomodate the
allocation.

This commit also adjusts GenerationFree() so that if we pfree the final
allocation in the current generation block, we now mark that block as
empty and keep it as the current block.  Previously we free'd that block
and set the current block to NULL.  Doing that meant we needed a special
case in GenerationAlloc to check if GenerationContext.block was NULL.
So this both reduces free/malloc calls and reduces the work done in
GenerationAlloc().

In passing, improve some comments in aset.c

Discussion: https://postgr.es/m/CAApHDvpHVSJqqb4B4OZLixr=CotKq-eKkbwZqvZVo_biYvUvQA@mail.gmail.com

1 files changed, 141 insertions, 91 deletions

diff --git a/src/backend/utils/mmgr/slab.c b/src/backend/utils/mmgr/slab.c
index bc91446cb3a..516e1c95aaf 100644
--- a/src/backend/utils/mmgr/slab.c
+++ b/src/backend/utils/mmgr/slab.c
@@ -491,9 +491,140 @@ SlabDelete(MemoryContext context)
 }
 
 /*
+ * Small helper for allocating a new chunk from a chunk, to avoid duplicating
+ * the code between SlabAlloc() and SlabAllocFromNewBlock().
+ */
+static inline void *
+SlabAllocSetupNewChunk(MemoryContext context, SlabBlock *block,
+					   MemoryChunk *chunk, Size size)
+{
+	SlabContext *slab = (SlabContext *) context;
+
+	/*
+	 * Check that the chunk pointer is actually somewhere on the block and is
+	 * aligned as expected.
+	 */
+	Assert(chunk >= SlabBlockGetChunk(slab, block, 0));
+	Assert(chunk <= SlabBlockGetChunk(slab, block, slab->chunksPerBlock - 1));
+	Assert(SlabChunkMod(slab, block, chunk) == 0);
+
+	/* Prepare to initialize the chunk header. */
+	VALGRIND_MAKE_MEM_UNDEFINED(chunk, Slab_CHUNKHDRSZ);
+
+	MemoryChunkSetHdrMask(chunk, block, MAXALIGN(slab->chunkSize), MCTX_SLAB_ID);
+
+#ifdef MEMORY_CONTEXT_CHECKING
+	/* slab mark to catch clobber of "unused" space */
+	Assert(slab->chunkSize < (slab->fullChunkSize - Slab_CHUNKHDRSZ));
+	set_sentinel(MemoryChunkGetPointer(chunk), size);
+	VALGRIND_MAKE_MEM_NOACCESS(((char *) chunk) + Slab_CHUNKHDRSZ +
+							   slab->chunkSize,
+							   slab->fullChunkSize -
+							   (slab->chunkSize + Slab_CHUNKHDRSZ));
+#endif
+
+#ifdef RANDOMIZE_ALLOCATED_MEMORY
+	/* fill the allocated space with junk */
+	randomize_mem((char *) MemoryChunkGetPointer(chunk), size);
+#endif
+
+	/* Disallow access to the chunk header. */
+	VALGRIND_MAKE_MEM_NOACCESS(chunk, Slab_CHUNKHDRSZ);
+
+	return MemoryChunkGetPointer(chunk);
+}
+
+pg_noinline
+static void *
+SlabAllocFromNewBlock(MemoryContext context, Size size, int flags)
+{
+	SlabContext *slab = (SlabContext *) context;
+	SlabBlock  *block;
+	MemoryChunk *chunk;
+	dlist_head *blocklist;
+	int			blocklist_idx;
+
+	/* to save allocating a new one, first check the empty blocks list */
+	if (dclist_count(&slab->emptyblocks) > 0)
+	{
+		dlist_node *node = dclist_pop_head_node(&slab->emptyblocks);
+
+		block = dlist_container(SlabBlock, node, node);
+
+		/*
+		 * SlabFree() should have left this block in a valid state with all
+		 * chunks free.  Ensure that's the case.
+		 */
+		Assert(block->nfree == slab->chunksPerBlock);
+
+		/* fetch the next chunk from this block */
+		chunk = SlabGetNextFreeChunk(slab, block);
+	}
+	else
+	{
+		block = (SlabBlock *) malloc(slab->blockSize);
+
+		if (unlikely(block == NULL))
+			return MemoryContextAllocationFailure(context, size, flags);
+
+		block->slab = slab;
+		context->mem_allocated += slab->blockSize;
+
+		/* use the first chunk in the new block */
+		chunk = SlabBlockGetChunk(slab, block, 0);
+
+		block->nfree = slab->chunksPerBlock - 1;
+		block->unused = SlabBlockGetChunk(slab, block, 1);
+		block->freehead = NULL;
+		block->nunused = slab->chunksPerBlock - 1;
+	}
+
+	/* find the blocklist element for storing blocks with 1 used chunk */
+	blocklist_idx = SlabBlocklistIndex(slab, block->nfree);
+	blocklist = &slab->blocklist[blocklist_idx];
+
+	/* this better be empty.  We just added a block thinking it was */
+	Assert(dlist_is_empty(blocklist));
+
+	dlist_push_head(blocklist, &block->node);
+
+	slab->curBlocklistIndex = blocklist_idx;
+
+	return SlabAllocSetupNewChunk(context, block, chunk, size);
+}
+
+/*
+ * SlabAllocInvalidSize
+ *		Handle raising an ERROR for an invalid size request.  We don't do this
+ *		in slab alloc as calling the elog functions would force the compiler
+ *		to setup the stack frame in SlabAlloc.  For performance reasons, we
+ *		want to avoid that.
+ */
+pg_noinline
+static void
+pg_attribute_noreturn()
+SlabAllocInvalidSize(MemoryContext context, Size size)
+{
+	SlabContext *slab = (SlabContext *) context;
+
+	elog(ERROR, "unexpected alloc chunk size %zu (expected %u)", size,
+		 slab->chunkSize);
+}
+
+/*
  * SlabAlloc
- *		Returns a pointer to allocated memory of given size or NULL if
- *		request could not be completed; memory is added to the slab.
+ *		Returns a pointer to a newly allocated memory chunk or raises an ERROR
+ *		on allocation failure, or returns NULL when flags contains
+ *		MCXT_ALLOC_NO_OOM.  'size' must be the same size as was specified
+ *		during SlabContextCreate().
+ *
+ * This function should only contain the most common code paths.  Everything
+ * else should be in pg_noinline helper functions, thus avoiding the overhead
+ * of creating a stack frame for the common cases.  Allocating memory is often
+ * a bottleneck in many workloads, so avoiding stack frame setup is
+ * worthwhile.  Helper functions should always directly return the newly
+ * allocated memory so that we can just return that address directly as a tail
+ * call.
  */
 void *
 SlabAlloc(MemoryContext context, Size size, int flags)
@@ -513,66 +644,16 @@ SlabAlloc(MemoryContext context, Size size, int flags)
 	 * MemoryContextCheckSize check.
 	 */
 	if (unlikely(size != slab->chunkSize))
-		elog(ERROR, "unexpected alloc chunk size %zu (expected %u)",
-			 size, slab->chunkSize);
+		SlabAllocInvalidSize(context, size);
 
-	/*
-	 * Handle the case when there are no partially filled blocks available.
-	 * SlabFree() will have updated the curBlocklistIndex setting it to zero
-	 * to indicate that it has freed the final block.  Also later in
-	 * SlabAlloc() we will set the curBlocklistIndex to zero if we end up
-	 * filling the final block.
-	 */
 	if (unlikely(slab->curBlocklistIndex == 0))
 	{
-		dlist_head *blocklist;
-		int			blocklist_idx;
-
-		/* to save allocating a new one, first check the empty blocks list */
-		if (dclist_count(&slab->emptyblocks) > 0)
-		{
-			dlist_node *node = dclist_pop_head_node(&slab->emptyblocks);
-
-			block = dlist_container(SlabBlock, node, node);
-
-			/*
-			 * SlabFree() should have left this block in a valid state with
-			 * all chunks free.  Ensure that's the case.
-			 */
-			Assert(block->nfree == slab->chunksPerBlock);
-
-			/* fetch the next chunk from this block */
-			chunk = SlabGetNextFreeChunk(slab, block);
-		}
-		else
-		{
-			block = (SlabBlock *) malloc(slab->blockSize);
-
-			if (unlikely(block == NULL))
-				return MemoryContextAllocationFailure(context, size, flags);
-
-			block->slab = slab;
-			context->mem_allocated += slab->blockSize;
-
-			/* use the first chunk in the new block */
-			chunk = SlabBlockGetChunk(slab, block, 0);
-
-			block->nfree = slab->chunksPerBlock - 1;
-			block->unused = SlabBlockGetChunk(slab, block, 1);
-			block->freehead = NULL;
-			block->nunused = slab->chunksPerBlock - 1;
-		}
-
-		/* find the blocklist element for storing blocks with 1 used chunk */
-		blocklist_idx = SlabBlocklistIndex(slab, block->nfree);
-		blocklist = &slab->blocklist[blocklist_idx];
-
-		/* this better be empty.  We just added a block thinking it was */
-		Assert(dlist_is_empty(blocklist));
-
-		dlist_push_head(blocklist, &block->node);
-
-		slab->curBlocklistIndex = blocklist_idx;
+		/*
+		 * Handle the case when there are no partially filled blocks
+		 * available.  This happens either when the last allocation took the
+		 * last chunk in the block, or when SlabFree() free'd the final block.
+		 */
+		return SlabAllocFromNewBlock(context, size, flags);
 	}
 	else
 	{
@@ -609,38 +690,7 @@ SlabAlloc(MemoryContext context, Size size, int flags)
 		}
 	}
 
-	/*
-	 * Check that the chunk pointer is actually somewhere on the block and is
-	 * aligned as expected.
-	 */
-	Assert(chunk >= SlabBlockGetChunk(slab, block, 0));
-	Assert(chunk <= SlabBlockGetChunk(slab, block, slab->chunksPerBlock - 1));
-	Assert(SlabChunkMod(slab, block, chunk) == 0);
-
-	/* Prepare to initialize the chunk header. */
-	VALGRIND_MAKE_MEM_UNDEFINED(chunk, Slab_CHUNKHDRSZ);
-
-	MemoryChunkSetHdrMask(chunk, block, MAXALIGN(slab->chunkSize),
-						  MCTX_SLAB_ID);
-#ifdef MEMORY_CONTEXT_CHECKING
-	/* slab mark to catch clobber of "unused" space */
-	Assert(slab->chunkSize < (slab->fullChunkSize - Slab_CHUNKHDRSZ));
-	set_sentinel(MemoryChunkGetPointer(chunk), size);
-	VALGRIND_MAKE_MEM_NOACCESS(((char *) chunk) +
-							   Slab_CHUNKHDRSZ + slab->chunkSize,
-							   slab->fullChunkSize -
-							   (slab->chunkSize + Slab_CHUNKHDRSZ));
-#endif
-
-#ifdef RANDOMIZE_ALLOCATED_MEMORY
-	/* fill the allocated space with junk */
-	randomize_mem((char *) MemoryChunkGetPointer(chunk), size);
-#endif
-
-	/* Disallow access to the chunk header. */
-	VALGRIND_MAKE_MEM_NOACCESS(chunk, Slab_CHUNKHDRSZ);
-
-	return MemoryChunkGetPointer(chunk);
+	return SlabAllocSetupNewChunk(context, block, chunk, size);
 }
 
 /*