Fixed bug CORE-6043 : GTTs do not release used space

builds/install/misc/firebird.conf.in

@@ -607,6 +607,26 @@
 #BugcheckAbort = 0
 
 
+# ----------------------------
+# How COMMIT\ROLLBACK RETAINING handle GLOBAL TEMPORARY TABLE ON COMMIT DELETE ROWS 
+#
+# GLOBAL TEMPORARY TABLE ON COMMIT DELETE ROWS (GTT) data is cleared on "hard"
+# commit (rollback) but should be preserved on commit(rollback) retaining. 
+# Due to bug in initial implementation in Firebird 2.1 that data is not visible
+# for the application. This setting allows to preserve backward compatibility 
+# (i.e. clear GTT data on commit\rollback retaining).
+# Value of 0 makes engine to not clear GTT data on commit\rollback retaining and
+# let application to see it.
+# Default value is 0 (preserve GTT data on commit\rollback retaining).
+# Note: this setting is expected to be removed at Firebird 5.
+#
+# Per-database configurable.
+#
+# Type: boolean
+#
+#ClearGTTAtRetaining = 0
+
+
 # ----------------------------
 # Relaxing relation alias checking rules in SQL
 #

doc/sql.extensions/README.global_temporary_tables

@@ -67,6 +67,11 @@ database of course). This is much quicker than traditional row by row delete +
 garbage collection of deleted record versions. DELETE triggers are not fired in 
 this case.
 
+  Note, COMMIT\ROLLBACK RETAINING preserves data of GTT with ON COMMIT DELETE ROWS.
+Due to bug in Firebird 2.x that data was not visible to the user application.
+It is fixed in Firebird 3. See also description of ClearGTTAtRetaining setting at 
+firebird.conf.
+
 	Data and index pages of all of the GTTs instances are placed in separate temporary 
 files. Each connection has its own temporary file created when this connection 
 first referenced some GTT. Also these temporary files are always opened with "Forced 

src/common/config/config.cpp

@@ -227,7 +227,8 @@ const Config::ConfigEntry Config::entries[MAX_CONFIG_KEY] =
 	{TYPE_INTEGER,		"ExtConnPoolLifeTime",		(ConfigValue) 7200},
 	{TYPE_INTEGER,		"SnapshotsMemSize",			(ConfigValue) 65536}, // bytes
 	{TYPE_INTEGER,		"TipCacheBlockSize",		(ConfigValue) 4194304}, // bytes
-	{TYPE_BOOLEAN,		"ReadConsistency",			(ConfigValue) true}
+	{TYPE_BOOLEAN,		"ReadConsistency",			(ConfigValue) true},
+	{TYPE_BOOLEAN,		"ClearGTTAtRetaining",		(ConfigValue) false}
 };
 
 /******************************************************************************
@@ -887,3 +888,8 @@ bool Config::getReadConsistency() const
 {
 	return get<bool>(KEY_READ_CONSISTENCY);
 }
+
+bool Config::getClearGTTAtRetaining() const
+{
+	return get<bool>(KEY_CLEAR_GTT_RETAINING);
+}

src/common/config/config.h

@@ -154,6 +154,7 @@ public:
 		KEY_SNAPSHOTS_MEM_SIZE,
 		KEY_TIP_CACHE_BLOCK_SIZE,
 		KEY_READ_CONSISTENCY,
+		KEY_CLEAR_GTT_RETAINING,
 		MAX_CONFIG_KEY		// keep it last
 	};
 
@@ -387,6 +388,8 @@ public:
 	ULONG getTipCacheBlockSize() const;
 
 	bool getReadConsistency() const;
+
+	bool getClearGTTAtRetaining() const;
 };
 
 // Implementation of interface to access master configuration file

src/jrd/Relation.cpp

@@ -196,6 +196,26 @@ bool jrd_rel::delPages(thread_db* tdbb, TraNumber tran, RelationPages* aPages)
 	return true;
 }
 
+void jrd_rel::retainPages(thread_db* tdbb, TraNumber oldNumber, TraNumber newNumber)
+{
+	fb_assert(rel_flags & REL_temp_tran);
+	fb_assert(oldNumber != 0);
+	fb_assert(newNumber != 0);
+
+	SINT64 inst_id = oldNumber;
+	FB_SIZE_T pos;
+	if (!rel_pages_inst->find(oldNumber, pos))
+		return;
+
+	RelationPages* pages = (*rel_pages_inst)[pos];
+	fb_assert(pages->rel_instance_id == oldNumber);
+
+	rel_pages_inst->remove(pos);
+
+	pages->rel_instance_id = newNumber;
+	rel_pages_inst->add(pages);
+}
+
 void jrd_rel::getRelLockKey(thread_db* tdbb, UCHAR* key)
 {
 	const ULONG val = rel_id;

src/jrd/Relation.h

@@ -279,6 +279,7 @@ public:
 	}
 
 	bool	delPages(thread_db* tdbb, TraNumber tran = MAX_TRA_NUMBER, RelationPages* aPages = NULL);
+	void	retainPages(thread_db* tdbb, TraNumber oldNumber, TraNumber newNumber);
 
 	void	getRelLockKey(thread_db* tdbb, UCHAR* key);
 	USHORT	getRelLockKeyLength() const;

src/jrd/cch.cpp

@@ -138,6 +138,7 @@ static LatchState latch_buffer(thread_db*, Sync&, BufferDesc*, const PageNumber,
 static LockState lock_buffer(thread_db*, BufferDesc*, const SSHORT, const SCHAR);
 static ULONG memory_init(thread_db*, BufferControl*, SLONG);
 static void page_validation_error(thread_db*, win*, SSHORT);
+static void purgePrecedence(BufferControl*, BufferDesc*);
 static SSHORT related(BufferDesc*, const BufferDesc*, SSHORT, const ULONG);
 static bool writeable(BufferDesc*);
 static bool is_writeable(BufferDesc*, const ULONG);
@@ -204,6 +205,82 @@ const int PRE_EXISTS		= -1;
 const int PRE_UNKNOWN		= -2;
 
 
+void CCH_clean_page(thread_db* tdbb, PageNumber page)
+{
+/**************************************
+ *  C C H _ c l e a n _ p a g e
+ **************************************
+ *
+ * Functional description
+ *  Clear dirty status and dependencies. Buffer must be unused.
+ *  If buffer with given page number is not found - it is OK, do nothing.
+ *  Used to remove dirty pages from cache after releasing temporary objects.
+ *
+ **************************************/
+	SET_TDBB(tdbb);
+	Database* dbb = tdbb->getDatabase();
+
+	fb_assert(page.getPageNum() > 0);
+	fb_assert(page.isTemporary());
+	if (!page.isTemporary())
+		return;
+
+	BufferControl* bcb = dbb->dbb_bcb;
+	BufferDesc* bdb = NULL;
+	{
+		Sync bcbSync(&bcb->bcb_syncObject, "CCH_clean_page");
+		bcbSync.lock(SYNC_SHARED);
+
+		bdb = find_buffer(bcb, page, false);
+		if (!bdb)
+			return;
+
+		fb_assert(bdb->bdb_use_count == 0);
+		if (!bdb->addRefConditional(tdbb, SYNC_EXCLUSIVE))
+			return;
+	}
+
+	// temporary pages should have no precedence relationship
+	if (!QUE_EMPTY(bdb->bdb_higher))
+		purgePrecedence(bcb, bdb);
+
+	fb_assert(QUE_EMPTY(bdb->bdb_higher));
+	fb_assert(QUE_EMPTY(bdb->bdb_lower));
+
+	if (!QUE_EMPTY(bdb->bdb_lower) || !QUE_EMPTY(bdb->bdb_higher))
+	{
+		bdb->release(tdbb, true);
+		return;
+	}
+
+	if (bdb->bdb_flags & (BDB_dirty | BDB_db_dirty))
+	{
+		bdb->bdb_difference_page = 0;
+		bdb->bdb_transactions = 0;
+		bdb->bdb_mark_transaction = 0;
+
+		if (!(bdb->bdb_bcb->bcb_flags & BCB_keep_pages))
+			removeDirty(dbb->dbb_bcb, bdb);
+
+		bdb->bdb_flags &= ~(BDB_must_write | BDB_system_dirty | BDB_db_dirty);
+		clear_dirty_flag_and_nbak_state(tdbb, bdb);
+	}
+
+	{
+		Sync lruSync(&bcb->bcb_syncLRU, "CCH_release");
+		lruSync.lock(SYNC_EXCLUSIVE);
+
+		if (bdb->bdb_flags & BDB_lru_chained)
+			requeueRecentlyUsed(bcb);
+
+		QUE_DELETE(bdb->bdb_in_use);
+		QUE_APPEND(bcb->bcb_in_use, bdb->bdb_in_use);
+	}
+
+	bdb->release(tdbb, true);
+}
+
+
 int CCH_down_grade_dbb(void* ast_object)
 {
 /**************************************

src/jrd/cch_proto.h

@@ -40,6 +40,7 @@ enum LockState {
 	lsError
 };
 
+void		CCH_clean_page(Jrd::thread_db*, Jrd::PageNumber);
 int			CCH_down_grade_dbb(void*);
 bool		CCH_exclusive(Jrd::thread_db*, USHORT, SSHORT, Firebird::Sync*);
 bool		CCH_exclusive_attachment(Jrd::thread_db*, USHORT, SSHORT, Firebird::Sync*);

src/jrd/dpm.epp

@@ -820,7 +820,7 @@ void DPM_delete( thread_db* tdbb, record_param* rpb, ULONG prior_page)
 
 	for (;;)
 	{
-		relPages = rpb->rpb_relation->getPages(tdbb, rpb->rpb_transaction_nr);
+		relPages = rpb->rpb_relation->getPages(tdbb);
 		pwindow = WIN(relPages->rel_pg_space_id, -1);
 
 		if (!(ppage = get_pointer_page(tdbb, rpb->rpb_relation, relPages, &pwindow,
@@ -3266,7 +3266,7 @@ static rhd* locate_space(thread_db* tdbb,
 	CHECK_DBB(dbb);
 
 	jrd_rel* relation = rpb->rpb_relation;
-	RelationPages* relPages = relation->getPages(tdbb, rpb->rpb_transaction_nr);
+	RelationPages* relPages = relation->getPages(tdbb);
 	WIN* window = &rpb->getWindow(tdbb);
 
 	// If there is a preferred page, try there first

src/jrd/pag.cpp

@@ -1622,6 +1622,12 @@ void PAG_release_pages(thread_db* tdbb, USHORT pageSpaceID, int cntRelease,
 		pageSpace->pipWithExtent.exchangeLower(sequence);
 
 	CCH_RELEASE(tdbb, &pip_window);
+
+	if (pageSpace->isTemporary())
+	{
+		for (int i = 0; i < cntRelease; i++)
+			CCH_clean_page(tdbb, PageNumber(pageSpaceID, pgNums[i]));
+	}
 }
 
 

src/jrd/tra.cpp

@@ -101,6 +101,8 @@ static tx_inv_page* fetch_inventory_page(thread_db*, WIN* window, ULONG sequence
 static const char* get_lockname_v3(const UCHAR lock);
 static ULONG inventory_page(thread_db*, ULONG);
 static int limbo_transaction(thread_db*, TraNumber id);
+static void release_temp_tables(thread_db*, jrd_tra*);
+static void retain_temp_tables(thread_db*, jrd_tra*, TraNumber);
 static void restart_requests(thread_db*, jrd_tra*);
 static void start_sweeper(thread_db*);
 static THREAD_ENTRY_DECLARE sweep_database(THREAD_ENTRY_PARAM);
@@ -1254,18 +1256,7 @@ void TRA_release_transaction(thread_db* tdbb, jrd_tra* transaction, Jrd::TraceTr
 		}
 	}
 
-	{ // scope
-		vec<jrd_rel*>& rels = *attachment->att_relations;
-
-		for (FB_SIZE_T i = 0; i < rels.count(); i++)
-		{
-			jrd_rel* relation = rels[i];
-
-			if (relation && (relation->rel_flags & REL_temp_tran))
-				relation->delPages(tdbb, transaction->tra_number);
-		}
-
-	} // end scope
+	release_temp_tables(tdbb, transaction);
 
 	// Release the locks associated with the transaction
 
@@ -1370,7 +1361,7 @@ void TRA_rollback(thread_db* tdbb, jrd_tra* transaction, const bool retaining_fl
 		Savepoint::destroy(transaction->tra_save_point);
 		fb_assert(!transaction->tra_save_point);
 	}
-	else
+	else if (!retaining_flag)
 	{
 		// Remove undo data for GTT ON COMMIT DELETE ROWS as their data will be released
 		// at transaction end anyway and we don't need to waste time backing it out
@@ -2432,6 +2423,57 @@ void jrd_tra::tra_abort(const char* reason)
 }
 
 
+static void release_temp_tables(thread_db* tdbb, jrd_tra* transaction)
+{
+/**************************************
+ *
+ *	r e l e a s e _ t e m p _ t a b l e s
+ *
+ **************************************
+ *
+ * Functional description
+ *	Release data of temporary tables with transaction lifetime
+ *
+ **************************************/
+	Attachment* att = tdbb->getAttachment();
+	vec<jrd_rel*>& rels = *att->att_relations;
+
+	for (FB_SIZE_T i = 0; i < rels.count(); i++)
+	{
+		jrd_rel* relation = rels[i];
+
+		if (relation && (relation->rel_flags & REL_temp_tran))
+			relation->delPages(tdbb, transaction->tra_number);
+	}
+}
+
+
+static void retain_temp_tables(thread_db* tdbb, jrd_tra* transaction, TraNumber new_number)
+{
+/**************************************
+ *
+ *	r e t a i n _ t e m p _ t a b l e s
+ *
+ **************************************
+ *
+ * Functional description
+ *	Reassign instance of temporary tables with transaction lifetime to the new 
+ *  transaction number (see retain_context).
+ *
+ **************************************/
+	Attachment* att = tdbb->getAttachment();
+	vec<jrd_rel*>& rels = *att->att_relations;
+
+	for (FB_SIZE_T i = 0; i < rels.count(); i++)
+	{
+		jrd_rel* relation = rels[i];
+
+		if (relation && (relation->rel_flags & REL_temp_tran))
+			relation->retainPages(tdbb, transaction->tra_number, new_number);
+	}
+}
+
+
 static void restart_requests(thread_db* tdbb, jrd_tra* trans)
 {
 /**************************************
@@ -2557,6 +2599,11 @@ static void retain_context(thread_db* tdbb, jrd_tra* transaction, bool commit, i
 		else
 			REPL_trans_rollback(tdbb, transaction);
 	}
+	if (dbb->dbb_config->getClearGTTAtRetaining())
+		release_temp_tables(tdbb, transaction);
+	else
+		retain_temp_tables(tdbb, transaction, new_number);
+
 	transaction->tra_number = new_number;
 
 	// Release transaction lock since it isn't needed