Fixed related issues:

CORE-1518 - Adding a non-null restricted column to a populated table renders the table inconsistent
CORE-2696 - "alter table" command can add a field which has "not null" definition
CORE-1748 - Unrestorable backup
CORE-1355 - Zero display instead of NULL

src/jrd/dfw.epp

@@ -95,6 +95,7 @@
 #include "../jrd/dpm_proto.h"
 #include "../jrd/dsc_proto.h"
 #include "../jrd/err_proto.h"
+#include "../jrd/evl_proto.h"
 #include "../jrd/exe_proto.h"
 #include "../jrd/ext_proto.h"
 #include "../jrd/gds_proto.h"
@@ -113,6 +114,7 @@
 #include "../jrd/scl_proto.h"
 #include "../jrd/sdw_proto.h"
 #include "../jrd/thread_proto.h"
+#include "../jrd/tra_proto.h"
 #include "../jrd/event_proto.h"
 #include "../jrd/nbak.h"
 #include "../jrd/trig.h"
@@ -4776,6 +4778,21 @@ static Format* make_format(thread_db* tdbb, jrd_rel* relation, USHORT* version,
 			*desc = tfb->tfb_desc;
 		if (tfb->tfb_flags & TFB_computed)
 			desc->dsc_dtype |= COMPUTED_FLAG;
+
+		format->fmt_defaults[tfb->tfb_id] = tfb->tfb_default;
+		format->fmt_defaults[tfb->tfb_id].vlu_misc = tfb->tfb_default.vlu_misc;
+
+		if (tfb->tfb_default.vlu_string)
+		{
+			fb_assert(format->fmt_defaults[tfb->tfb_id].vlu_desc.dsc_dtype == dtype_text);
+			format->fmt_defaults[tfb->tfb_id].vlu_desc.dsc_address =
+				format->fmt_defaults[tfb->tfb_id].vlu_string->str_data;
+		}
+		else
+		{
+			format->fmt_defaults[tfb->tfb_id].vlu_desc.dsc_address =
+				(UCHAR*) &format->fmt_defaults[tfb->tfb_id].vlu_misc;
+		}
 	}
 
 /* Compute the offsets of the various fields */
@@ -4847,18 +4864,19 @@ static Format* make_format(thread_db* tdbb, jrd_rel* relation, USHORT* version,
 		FMTS.RDB$RELATION_ID = relation->rel_id;
 		blb* blob = BLB_create(tdbb, dbb->dbb_sys_trans, &FMTS.RDB$DESCRIPTOR);
 
-		if (sizeof(Ods::Descriptor) == sizeof(struct dsc) || dbb->dbb_ods_version < ODS_VERSION11)
+		if (dbb->dbb_ods_version < ODS_VERSION11)
 		{
 			// For ODS10 and earlier put descriptors in their in-memory representation
 			// This makes 64-bit and 32-bit ODS10 different for the same architecture.
 			BLB_put_segment(tdbb, blob, reinterpret_cast<const UCHAR*>(&(format->fmt_desc[0])),
 							(USHORT)(format->fmt_count * sizeof(dsc)));
 		}
-		else {
+		else
+		{
 			// Use generic representation of formats with 32-bit offsets
 
 			Firebird::Array<Ods::Descriptor> odsDescs;
-			Ods::Descriptor *odsDesc = odsDescs.getBuffer(format->fmt_count);
+			Ods::Descriptor* odsDesc = odsDescs.getBuffer(format->fmt_count);
 
 			for (Format::fmt_desc_const_iterator desc = format->fmt_desc.begin();
 				 desc < format->fmt_desc.end(); ++desc, ++odsDesc)
@@ -4866,8 +4884,47 @@ static Format* make_format(thread_db* tdbb, jrd_rel* relation, USHORT* version,
 				*odsDesc = *desc;
 			}
 
-			BLB_put_segment(tdbb, blob, (UCHAR*) odsDescs.begin(),
+			HalfStaticArray<UCHAR, BUFFER_MEDIUM> buffer;
-							odsDescs.getCount() * sizeof(Ods::Descriptor));
+
+			if (dbb->dbb_ods_version >= ODS_VERSION12)
+			{
+				buffer.add(UCHAR(format->fmt_count));
+				buffer.add(UCHAR(format->fmt_count >> 8));
+			}
+
+			buffer.add((UCHAR*) odsDescs.begin(), odsDescs.getCount() * sizeof(Ods::Descriptor));
+
+			if (dbb->dbb_ods_version >= ODS_VERSION12)
+			{
+				size_t pos = buffer.getCount();
+				buffer.add(0);
+				buffer.add(0);
+
+				USHORT i = 0, dflCount = 0;
+				for (Format::fmt_defaults_iterator impure = format->fmt_defaults.begin();
+					 impure != format->fmt_defaults.end(); ++impure, ++i)
+				{
+					if (!impure->vlu_desc.isUnknown())
+					{
+						dsc desc = impure->vlu_desc;
+						desc.dsc_address = NULL;
+
+						Ods::Descriptor odsDflDesc = desc;
+
+						buffer.add(UCHAR(i));
+						buffer.add(UCHAR(i >> 8));
+						buffer.add((UCHAR*) &odsDflDesc, sizeof(odsDflDesc));
+						buffer.add(impure->vlu_desc.dsc_address, impure->vlu_desc.dsc_length);
+
+						++dflCount;
+					}
+				}
+
+				buffer[pos] = UCHAR(dflCount);
+				buffer[pos + 1] = UCHAR(dflCount >> 8);
+			}
+
+			BLB_put_segment(tdbb, blob, buffer.begin(), buffer.getCount());
 		}
 		BLB_close(tdbb, blob);
 	END_STORE;
@@ -4995,7 +5052,24 @@ static bool make_version(thread_db* tdbb, SSHORT phase, DeferredWork* work,
 								{
 									RFR.RDB$FIELD_ID = REL.RDB$FIELD_ID;
 									RFR.RDB$FIELD_ID.NULL = FALSE;
+
+									bool notNull = (RFR.RDB$NULL_FLAG.NULL ?
+										(FLD.RDB$NULL_FLAG.NULL ? false : (bool) FLD.RDB$NULL_FLAG) :
+										(bool) RFR.RDB$NULL_FLAG);
+
+									if (notNull)
+									{
+										dsc desc;
+										desc.makeText(strlen(REL.RDB$RELATION_NAME), CS_METADATA,
+											(UCHAR*) REL.RDB$RELATION_NAME);
+
+										DeferredWork* work = DFW_post_work(transaction,
+											dfw_check_not_null, &desc, 0);
+										Array<int>& ids = DFW_get_ids(work);
+										ids.add(RFR.RDB$FIELD_ID);
+									}
 								}
+
 								REL.RDB$FIELD_ID++;
 							}
 							if (RFR.RDB$UPDATE_FLAG.NULL)
@@ -5039,12 +5113,18 @@ static bool make_version(thread_db* tdbb, SSHORT phase, DeferredWork* work,
 										   fb_utils::name_length(RFR.RDB$BASE_FIELD));
 					}
 					put_summary_blob(tdbb, blob, RSR_missing_value, &FLD.RDB$MISSING_VALUE, transaction);
-					put_summary_blob(tdbb, blob, RSR_default_value,
+
-								(RFR.RDB$DEFAULT_VALUE.isEmpty() ?
+					bid* defaultValue = RFR.RDB$DEFAULT_VALUE.isEmpty() ?
-									&FLD.RDB$DEFAULT_VALUE :
+						&FLD.RDB$DEFAULT_VALUE : &RFR.RDB$DEFAULT_VALUE;
-									&RFR.RDB$DEFAULT_VALUE), transaction);
+
+					put_summary_blob(tdbb, blob, RSR_default_value, defaultValue, transaction);
 					put_summary_blob(tdbb, blob, RSR_validation_blr, &FLD.RDB$VALIDATION_BLR, transaction);
-					if (FLD.RDB$NULL_FLAG || RFR.RDB$NULL_FLAG)
+
+					bool notNull = (RFR.RDB$NULL_FLAG.NULL ?
+						(FLD.RDB$NULL_FLAG.NULL ? false : (bool) FLD.RDB$NULL_FLAG) :
+						(bool) RFR.RDB$NULL_FLAG);
+
+					if (notNull)
 					{
 						put_summary_record(tdbb, blob, RSR_field_not_null,
 											nonnull_validation_blr, sizeof(nonnull_validation_blr));
@@ -5061,6 +5141,67 @@ static bool make_version(thread_db* tdbb, SSHORT phase, DeferredWork* work,
 					tfb->tfb_next = stack;
 					stack = tfb;
 
+					memset(&tfb->tfb_default, 0, sizeof(tfb->tfb_default));
+
+					if (notNull && !defaultValue->isEmpty())
+					{
+						Jrd::ContextPoolHolder context(tdbb, dbb->createPool());
+						jrd_req* defaultRequest = NULL;
+						try
+						{
+							jrd_nod* defaultNode = MET_parse_blob(tdbb, relation, defaultValue,
+								NULL, &defaultRequest, false);
+
+							{	// scope
+								AutoSetRestore2<jrd_req*, thread_db> autoRequest(tdbb,
+									&thread_db::getRequest, &thread_db::setRequest, defaultRequest);
+
+								defaultRequest->req_timestamp.validate();
+
+								TRA_attach_request(transaction, defaultRequest);
+								dsc* result = EVL_expr(tdbb, defaultNode);
+								TRA_detach_request(defaultRequest);
+
+								if (result)
+								{
+									tfb->tfb_default.vlu_desc = *result;
+
+									if (result->isText() || result->isBlob())
+									{
+										UCHAR* ptr = NULL;
+										MoveBuffer buffer;
+										int len = MOV_make_string2(tdbb, result, CS_NONE, &ptr, buffer, true);
+
+										VaryingString* newString = FB_NEW_RPT(*dbb->dbb_permanent,
+											len) VaryingString();
+										newString->str_length = USHORT(len);
+										memcpy(newString->str_data, ptr, len);
+
+										tfb->tfb_default.vlu_string = newString;
+										tfb->tfb_default.vlu_desc.dsc_dtype = dtype_text;
+										tfb->tfb_default.vlu_desc.dsc_length = USHORT(len);
+										tfb->tfb_default.vlu_desc.dsc_address = newString->str_data;
+									}
+									else
+									{
+										memcpy(&tfb->tfb_default.vlu_misc, result->dsc_address,
+											result->dsc_length);
+										tfb->tfb_default.vlu_desc.dsc_address =
+											(UCHAR*) &tfb->tfb_default.vlu_misc;
+									}
+								}
+							}
+						}
+						catch (const Exception&)
+						{
+							if (defaultRequest)
+								CMP_release(tdbb, defaultRequest);
+							throw;
+						}
+
+						CMP_release(tdbb, defaultRequest);
+					}
+
 					/* for text data types, grab the CHARACTER_SET and
 					   COLLATION to give the type of international text */
 

src/jrd/evl.cpp

@@ -1247,6 +1247,29 @@ bool EVL_field(jrd_rel* relation, Record* record, USHORT id, dsc* desc)
 
 		if (record && record->rec_format && relation)
 		{
+			thread_db* tdbb = JRD_get_thread_data();
+			Database* dbb = tdbb->getDatabase();
+
+			if (dbb->dbb_ods_version >= ODS_VERSION12)
+			{
+				while (format &&
+					(id >= format->fmt_defaults.getCount() ||
+					 format->fmt_defaults[id].vlu_desc.isUnknown()))
+				{
+					if (format->fmt_version >= relation->rel_current_format->fmt_version)
+					{
+						format = NULL;
+						break;
+					}
+
+					format = MET_format(tdbb, relation, format->fmt_version + 1);
+				}
+
+				return format && !(*desc = format->fmt_defaults[id].vlu_desc).isUnknown();
+			}
+
+			// Legacy ODS logic
+
 			/* A database sweep does not scan a relation's metadata. However
 			 * the change to substitute a default value for a missing "not null"
 			 * field makes it necessary to reference the field block.

src/jrd/exe.h

@@ -191,74 +191,6 @@ public:
 
 const size_t asb_delta	= ((sizeof(AggregateSort) - sizeof(jrd_nod)) / sizeof (jrd_nod**));
 
-
-// Various structures in the impure area
-
-struct impure_state
-{
-	SSHORT sta_state;
-};
-
-struct impure_value
-{
-	dsc vlu_desc;
-	USHORT vlu_flags; // Computed/invariant flags
-	VaryingString* vlu_string;
-	union
-	{
-		UCHAR vlu_uchar;
-		SSHORT vlu_short;
-		SLONG vlu_long;
-		SINT64 vlu_int64;
-		SQUAD vlu_quad;
-		SLONG vlu_dbkey[2];
-		float vlu_float;
-		double vlu_double;
-		GDS_TIMESTAMP vlu_timestamp;
-		GDS_TIME vlu_sql_time;
-		GDS_DATE vlu_sql_date;
-		bid vlu_bid;
-		void* vlu_invariant; // Pre-compiled invariant object for nod_like and other string functions
-	} vlu_misc;
-
-	void make_long(const SLONG val, const signed char scale = 0);
-	void make_int64(const SINT64 val, const signed char scale = 0);
-
-};
-
-// Do not use these methods where dsc_sub_type is not explicitly set to zero.
-inline void impure_value::make_long(const SLONG val, const signed char scale)
-{
-	this->vlu_misc.vlu_long = val;
-	this->vlu_desc.dsc_dtype = dtype_long;
-	this->vlu_desc.dsc_length = sizeof(SLONG);
-	this->vlu_desc.dsc_scale = scale;
-	this->vlu_desc.dsc_sub_type = 0;
-	this->vlu_desc.dsc_address = reinterpret_cast<UCHAR*>(&this->vlu_misc.vlu_long);
-}
-
-inline void impure_value::make_int64(const SINT64 val, const signed char scale)
-{
-	this->vlu_misc.vlu_int64 = val;
-	this->vlu_desc.dsc_dtype = dtype_int64;
-	this->vlu_desc.dsc_length = sizeof(SINT64);
-	this->vlu_desc.dsc_scale = scale;
-	this->vlu_desc.dsc_sub_type = 0;
-	this->vlu_desc.dsc_address = reinterpret_cast<UCHAR*>(&this->vlu_misc.vlu_int64);
-}
-
-
-struct impure_value_ex : public impure_value
-{
-	SLONG vlux_count;
-	blb* vlu_blob;
-};
-
-
-const int VLU_computed	= 1;	// An invariant sub-query has been computed
-const int VLU_null		= 2;	// An invariant sub-query computed to null
-const int VLU_checked	= 4;	// Constraint already checked in first read or assignment to argument/variable
-
 // Inversion (i.e. nod_index) impure area
 
 struct impure_inversion

src/jrd/jrd.h

@@ -936,16 +936,6 @@ typedef Firebird::HalfStaticArray<UCHAR, 256> MoveBuffer;
 
 } //namespace Jrd
 
-// Random string block -- as long as impure areas don't have
-// constructors and destructors, the need this varying string
-
-class VaryingString : public pool_alloc_rpt<SCHAR, type_str>
-{
-public:
-	USHORT str_length;
-	UCHAR str_data[2];			// one byte for ALLOC and one for the NULL
-};
-
 // Threading macros
 
 /* Define JRD_get_thread_data off the platform specific version.

src/jrd/met.epp

@@ -1207,7 +1207,7 @@ Format* MET_format(thread_db* tdbb, jrd_rel* relation, USHORT number)
 		}
 		blb* blob = BLB_open(tdbb, dbb->dbb_sys_trans, &X.RDB$DESCRIPTOR);
 
-		if (sizeof(Ods::Descriptor) == sizeof(struct dsc) || dbb->dbb_ods_version < ODS_VERSION11)
+		if (dbb->dbb_ods_version < ODS_VERSION11)
 		{
 			// For ODS10 and earlier read descriptors in their in-memory representation
 			// This makes 64-bit and 32-bit ODS10 different for the same architecture.
@@ -1231,11 +1231,24 @@ Format* MET_format(thread_db* tdbb, jrd_rel* relation, USHORT number)
 		{
 			// Use generic representation of formats with 32-bit offsets
 
-			const USHORT count = blob->blb_length / sizeof(Ods::Descriptor);
+			HalfStaticArray<UCHAR, BUFFER_MEDIUM> buffer;
+			BLB_get_data(tdbb, blob, buffer.getBuffer(blob->blb_length), blob->blb_length);
+			unsigned bufferPos = 0;
+			USHORT count;
+
+			if (dbb->dbb_ods_version >= ODS_VERSION12)
+			{
+				count = buffer[0] | (buffer[1] << 8);
+				bufferPos = 2;
+			}
+			else
+				count = buffer.getCount() / sizeof(Ods::Descriptor);
+
 			format = Format::newFormat(*dbb->dbb_permanent, count);
+
 			Firebird::Array<Ods::Descriptor> odsDescs;
-			Ods::Descriptor *odsDesc = odsDescs.getBuffer(count);
+			Ods::Descriptor* odsDesc = odsDescs.getBuffer(count);
-			BLB_get_data(tdbb, blob, (UCHAR*) odsDesc, blob->blb_length);
+			memcpy(odsDesc, buffer.begin() + bufferPos, count * sizeof(Ods::Descriptor));
 
 			for (Format::fmt_desc_iterator desc = format->fmt_desc.begin();
 				 desc < format->fmt_desc.end(); ++desc, ++odsDesc)
@@ -1244,6 +1257,40 @@ Format* MET_format(thread_db* tdbb, jrd_rel* relation, USHORT number)
 				if (odsDesc->dsc_offset)
 					format->fmt_length = odsDesc->dsc_offset + desc->dsc_length;
 			}
+
+			if (dbb->dbb_ods_version >= ODS_VERSION12)
+			{
+				UCHAR* p = buffer.begin() + bufferPos + count * sizeof(Ods::Descriptor);
+				count = p[0] | (p[1] << 8);
+				p += 2;
+
+				while (count-- > 0)
+				{
+					USHORT offset = p[0] | (p[1] << 8);
+					p += 2;
+
+					Ods::Descriptor* odsDflDesc = (Ods::Descriptor*) p;
+					p = (UCHAR*) (odsDflDesc + 1);
+
+					dsc& desc = format->fmt_defaults[offset].vlu_desc;
+					desc = *odsDflDesc;
+
+					if (desc.dsc_dtype == dtype_text)
+					{
+						VaryingString* vluString = FB_NEW_RPT(*dbb->dbb_permanent,
+							desc.dsc_length) VaryingString();
+						vluString->str_length = desc.dsc_length;
+						format->fmt_defaults[offset].vlu_string = vluString;
+						desc.dsc_address = format->fmt_defaults[offset].vlu_string->str_data;
+					}
+					else
+						desc.dsc_address = (UCHAR*) &format->fmt_defaults[offset].vlu_misc;
+
+					memcpy(desc.dsc_address, p, desc.dsc_length);
+
+					p += desc.dsc_length;
+				}
+			}
 		}
 	END_FOR;
 

src/jrd/met.h

@@ -58,9 +58,10 @@ class TemporaryField : public pool_alloc<type_tfb>
 {
 public:
 	TemporaryField*	tfb_next;		/* next block in chain */
-	USHORT			tfb_id;				/* id of field in relation */
+	USHORT			tfb_id;			/* id of field in relation */
 	USHORT			tfb_flags;
-	DSC				tfb_desc;
+	dsc				tfb_desc;
+	Jrd::impure_value	tfb_default;
 };
 
 // tfb_flags

src/jrd/val.h

@@ -34,11 +34,22 @@
 #include "../common/classes/MetaName.h"
 #include "../common/classes/QualifiedName.h"
 
+#include "../jrd/blb.h"
 #include "../jrd/dsc.h"
 #include "../jrd/ExtEngineManager.h"
 
 #define FLAG_BYTES(n)	(((n + BITS_PER_LONG) & ~((ULONG)BITS_PER_LONG - 1)) >> 3)
 
+// Random string block -- as long as impure areas don't have
+// constructors and destructors, the need this varying string
+
+class VaryingString : public pool_alloc_rpt<SCHAR, type_str>
+{
+public:
+	USHORT str_length;
+	UCHAR str_data[2];			// one byte for ALLOC and one for the NULL
+};
+
 const UCHAR DEFAULT_DOUBLE  = dtype_double;
 const ULONG MAX_FORMAT_SIZE	= 65535;
 
@@ -50,14 +61,96 @@ class jrd_req;
 class jrd_tra;
 class Symbol;
 
+// Various structures in the impure area
+
+struct impure_state
+{
+	SSHORT sta_state;
+};
+
+struct impure_value
+{
+	dsc vlu_desc;
+	USHORT vlu_flags; // Computed/invariant flags
+	VaryingString* vlu_string;
+	union
+	{
+		UCHAR vlu_uchar;
+		SSHORT vlu_short;
+		SLONG vlu_long;
+		SINT64 vlu_int64;
+		SQUAD vlu_quad;
+		SLONG vlu_dbkey[2];
+		float vlu_float;
+		double vlu_double;
+		GDS_TIMESTAMP vlu_timestamp;
+		GDS_TIME vlu_sql_time;
+		GDS_DATE vlu_sql_date;
+		bid vlu_bid;
+		void* vlu_invariant; // Pre-compiled invariant object for nod_like and other string functions
+	} vlu_misc;
+
+	void make_long(const SLONG val, const signed char scale = 0);
+	void make_int64(const SINT64 val, const signed char scale = 0);
+};
+
+// Do not use these methods where dsc_sub_type is not explicitly set to zero.
+inline void impure_value::make_long(const SLONG val, const signed char scale)
+{
+	this->vlu_misc.vlu_long = val;
+	this->vlu_desc.dsc_dtype = dtype_long;
+	this->vlu_desc.dsc_length = sizeof(SLONG);
+	this->vlu_desc.dsc_scale = scale;
+	this->vlu_desc.dsc_sub_type = 0;
+	this->vlu_desc.dsc_address = reinterpret_cast<UCHAR*>(&this->vlu_misc.vlu_long);
+}
+
+inline void impure_value::make_int64(const SINT64 val, const signed char scale)
+{
+	this->vlu_misc.vlu_int64 = val;
+	this->vlu_desc.dsc_dtype = dtype_int64;
+	this->vlu_desc.dsc_length = sizeof(SINT64);
+	this->vlu_desc.dsc_scale = scale;
+	this->vlu_desc.dsc_sub_type = 0;
+	this->vlu_desc.dsc_address = reinterpret_cast<UCHAR*>(&this->vlu_misc.vlu_int64);
+}
+
+struct impure_value_ex : public impure_value
+{
+	SLONG vlux_count;
+	blb* vlu_blob;
+};
+
+const int VLU_computed	= 1;	// An invariant sub-query has been computed
+const int VLU_null		= 2;	// An invariant sub-query computed to null
+const int VLU_checked	= 4;	// Constraint already checked in first read or assignment to argument/variable
+
+
 class Format : public pool_alloc<type_fmt>
 {
 public:
 	Format(MemoryPool& p, int len)
-		: fmt_count(len), fmt_desc(p, fmt_count)
+		: fmt_count(len), fmt_desc(p, fmt_count), fmt_defaults(p, fmt_count)
 	{
 		fmt_desc.resize(fmt_count);
+		fmt_defaults.resize(fmt_count);
+
+		for (fmt_defaults_iterator impure = fmt_defaults.begin();
+			 impure != fmt_defaults.end(); ++impure)
+		{
+			memset(&*impure, 0, sizeof(*impure));
+		}
 	}
+
+	~Format()
+	{
+		for (fmt_defaults_iterator impure = fmt_defaults.begin();
+			 impure != fmt_defaults.end(); ++impure)
+		{
+			delete impure->vlu_string;
+		}
+	}
+
 	static Format* newFormat(MemoryPool& p, int len = 0)
 	{
 		return FB_NEW(p) Format(p, len);
@@ -67,8 +160,12 @@ public:
 	USHORT fmt_count;
 	USHORT fmt_version;
 	Firebird::Array<dsc> fmt_desc;
+	Firebird::Array<impure_value> fmt_defaults;
+
 	typedef Firebird::Array<dsc>::iterator fmt_desc_iterator;
 	typedef Firebird::Array<dsc>::const_iterator fmt_desc_const_iterator;
+
+	typedef Firebird::Array<impure_value>::iterator fmt_defaults_iterator;
 };