Fast ALTER TABLE ADD COLUMN with a non-NULL default

Currently adding a column to a table with a non-NULL default results in
a rewrite of the table. For large tables this can be both expensive and
disruptive. This patch removes the need for the rewrite as long as the
default value is not volatile. The default expression is evaluated at
the time of the ALTER TABLE and the result stored in a new column
(attmissingval) in pg_attribute, and a new column (atthasmissing) is set
to true. Any existing row when fetched will be supplied with the
attmissingval. New rows will have the supplied value or the default and
so will never need the attmissingval.

Any time the table is rewritten all the atthasmissing and attmissingval
settings for the attributes are cleared, as they are no longer needed.

The most visible code change from this is in heap_attisnull, which
acquires a third TupleDesc argument, allowing it to detect a missing
value if there is one. In many cases where it is known that there will
not be any (e.g.  catalog relations) NULL can be passed for this
argument.

Andrew Dunstan, heavily modified from an original patch from Serge
Rielau.
Reviewed by Tom Lane, Andres Freund, Tomas Vondra and David Rowley.

Discussion: https://postgr.es/m/[email protected]

6 files changed, 61 insertions, 13 deletions

diff --git a/src/include/access/htup_details.h b/src/include/access/htup_details.h
index 67342ef63dc..cebaea097d1 100644
--- a/src/include/access/htup_details.h
+++ b/src/include/access/htup_details.h
@@ -799,7 +799,7 @@ extern void heap_fill_tuple(TupleDesc tupleDesc,
 				Datum *values, bool *isnull,
 				char *data, Size data_size,
 				uint16 *infomask, bits8 *bit);
-extern bool heap_attisnull(HeapTuple tup, int attnum);
+extern bool heap_attisnull(HeapTuple tup, int attnum, TupleDesc tupleDesc);
 extern Datum nocachegetattr(HeapTuple tup, int attnum,
 			   TupleDesc att);
 extern Datum heap_getsysattr(HeapTuple tup, int attnum, TupleDesc tupleDesc,
@@ -830,5 +830,7 @@ extern MinimalTuple heap_copy_minimal_tuple(MinimalTuple mtup);
 extern HeapTuple heap_tuple_from_minimal_tuple(MinimalTuple mtup);
 extern MinimalTuple minimal_tuple_from_heap_tuple(HeapTuple htup);
 extern size_t varsize_any(void *p);
+extern HeapTuple heap_expand_tuple(HeapTuple sourceTuple, TupleDesc tupleDesc);
+extern MinimalTuple minimal_expand_tuple(HeapTuple sourceTuple, TupleDesc tupleDesc);
 
 #endif							/* HTUP_DETAILS_H */
diff --git a/src/include/access/tupdesc.h b/src/include/access/tupdesc.h
index 415efbab97e..708160f645e 100644
--- a/src/include/access/tupdesc.h
+++ b/src/include/access/tupdesc.h
@@ -25,6 +25,8 @@ typedef struct attrDefault
 	char	   *adbin;			/* nodeToString representation of expr */
 } AttrDefault;
 
+typedef struct attrMissing *MissingPtr;
+
 typedef struct constrCheck
 {
 	char	   *ccname;
@@ -38,6 +40,7 @@ typedef struct tupleConstr
 {
 	AttrDefault *defval;		/* array */
 	ConstrCheck *check;			/* array */
+	MissingPtr	missing;		/* missing attributes values, NULL if none */
 	uint16		num_defval;
 	uint16		num_check;
 	bool		has_not_null;
diff --git a/src/include/access/tupdesc_details.h b/src/include/access/tupdesc_details.h
new file mode 100644
index 00000000000..741e996b3cc
--- /dev/null
+++ b/src/include/access/tupdesc_details.h
@@ -0,0 +1,29 @@
+/*-------------------------------------------------------------------------
+ *
+ * tupdesc_details.h
+ *	  POSTGRES tuple descriptor definitions we can't include everywhere
+ *
+ *
+ * Portions Copyright (c) 1996-2018, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/include/access/tupdesc_details.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef TUPDESC_DETAILS_H
+#define TUPDESC_DETAILS_H
+
+/*
+ * Structure used to represent value to be used when the attribute is not
+ * present at all in a tuple, i.e. when the column was created after the tuple
+ */
+
+typedef struct attrMissing
+{
+	bool		ammissingPresent;	/* true if non-NULL missing value exists */
+	Datum		ammissing;		/* value when attribute is missing */
+} AttrMissing;
+
+#endif							/* TUPDESC_DETAILS_H */
diff --git a/src/include/catalog/heap.h b/src/include/catalog/heap.h
index 3308fa3dfd5..59fc0524947 100644
--- a/src/include/catalog/heap.h
+++ b/src/include/catalog/heap.h
@@ -23,6 +23,7 @@ typedef struct RawColumnDefault
 {
 	AttrNumber	attnum;			/* attribute to attach default to */
 	Node	   *raw_default;	/* default value (untransformed parse tree) */
+	bool		missingMode;	/* true if part of add column processing */
 } RawColumnDefault;
 
 typedef struct CookedConstraint
@@ -103,8 +104,11 @@ extern List *AddRelationNewConstraints(Relation rel,
 						  bool is_local,
 						  bool is_internal);
 
+extern void RelationClearMissing(Relation rel);
+
 extern Oid StoreAttrDefault(Relation rel, AttrNumber attnum,
-				 Node *expr, bool is_internal);
+				 Node *expr, bool is_internal,
+				 bool add_column_mode);
 
 extern Node *cookDefault(ParseState *pstate,
 			Node *raw_default,
diff --git a/src/include/catalog/pg_attribute.h b/src/include/catalog/pg_attribute.h
index 8159383834d..5bb64f7c31d 100644
--- a/src/include/catalog/pg_attribute.h
+++ b/src/include/catalog/pg_attribute.h
@@ -133,6 +133,9 @@ CATALOG(pg_attribute,1249) BKI_BOOTSTRAP BKI_WITHOUT_OIDS BKI_ROWTYPE_OID(75) BK
 	/* Has DEFAULT value or not */
 	bool		atthasdef BKI_DEFAULT(f);
 
+	/* Has a missing value or not */
+	bool		atthasmissing BKI_DEFAULT(f);
+
 	/* One of the ATTRIBUTE_IDENTITY_* constants below, or '\0' */
 	char		attidentity BKI_DEFAULT("");
 
@@ -167,6 +170,12 @@ CATALOG(pg_attribute,1249) BKI_BOOTSTRAP BKI_WITHOUT_OIDS BKI_ROWTYPE_OID(75) BK
 
 	/* Column-level FDW options */
 	text		attfdwoptions[1] BKI_DEFAULT(_null_);
+
+	/*
+	 * Missing value for added columns. This is a one element array which lets
+	 * us store a value of the attribute type here.
+	 */
+	anyarray	attmissingval BKI_DEFAULT(_null_);
 #endif
 } FormData_pg_attribute;
 
@@ -191,7 +200,7 @@ typedef FormData_pg_attribute *Form_pg_attribute;
  * ----------------
  */
 
-#define Natts_pg_attribute				22
+#define Natts_pg_attribute				24
 #define Anum_pg_attribute_attrelid		1
 #define Anum_pg_attribute_attname		2
 #define Anum_pg_attribute_atttypid		3
@@ -206,15 +215,16 @@ typedef FormData_pg_attribute *Form_pg_attribute;
 #define Anum_pg_attribute_attalign		12
 #define Anum_pg_attribute_attnotnull	13
 #define Anum_pg_attribute_atthasdef		14
-#define Anum_pg_attribute_attidentity	15
-#define Anum_pg_attribute_attisdropped	16
-#define Anum_pg_attribute_attislocal	17
-#define Anum_pg_attribute_attinhcount	18
-#define Anum_pg_attribute_attcollation	19
-#define Anum_pg_attribute_attacl		20
-#define Anum_pg_attribute_attoptions	21
-#define Anum_pg_attribute_attfdwoptions 22
-
+#define Anum_pg_attribute_atthasmissing	15
+#define Anum_pg_attribute_attidentity	16
+#define Anum_pg_attribute_attisdropped	17
+#define Anum_pg_attribute_attislocal	18
+#define Anum_pg_attribute_attinhcount	19
+#define Anum_pg_attribute_attcollation	20
+#define Anum_pg_attribute_attacl		21
+#define Anum_pg_attribute_attoptions	22
+#define Anum_pg_attribute_attfdwoptions	23
+#define Anum_pg_attribute_attmissingval	24
 
 /* ----------------
  *		initial contents of pg_attribute
diff --git a/src/include/catalog/pg_class.h b/src/include/catalog/pg_class.h
index 85cdb99f1f6..135f33d0f30 100644
--- a/src/include/catalog/pg_class.h
+++ b/src/include/catalog/pg_class.h
@@ -149,7 +149,7 @@ typedef FormData_pg_class *Form_pg_class;
  */
 DATA(insert OID = 1247 (  pg_type		PGNSP 71 0 PGUID 0 0 0 0 0 0 0 f f p r 30 0 t f f f f f t n f 0 3 1 _null_ _null_ _null_));
 DESCR("");
-DATA(insert OID = 1249 (  pg_attribute	PGNSP 75 0 PGUID 0 0 0 0 0 0 0 f f p r 22 0 f f f f f f t n f 0 3 1 _null_ _null_ _null_));
+DATA(insert OID = 1249 (  pg_attribute	PGNSP 75 0 PGUID 0 0 0 0 0 0 0 f f p r 24 0 f f f f f f t n f 0 3 1 _null_ _null_ _null_));
 DESCR("");
 DATA(insert OID = 1255 (  pg_proc		PGNSP 81 0 PGUID 0 0 0 0 0 0 0 f f p r 28 0 t f f f f f t n f 0 3 1 _null_ _null_ _null_));
 DESCR("");