Add AUTO RELEASE TEMP BLOBID transaction option (#8323)

* Add AUTO RELEASE TEMP BLOBID transaction option

It makes the transaction release temporary ID of user BLOB just after its materialization. It's useful for massive insertions of records with user-defined BLOBs because it eliminates the memory overhead caused by creating and keeping temporary IDs until the transaction ends. This option is used during the database restore.

* Place tokens in the correct sections

* Avoid repeated attempts to start a transaction with options that are not supported by the target server

* Correct AUTO RELEASE TEMP BLOBID description

* Check bad_tpb_form error for more reliable detection of unsupported transaction options

* Do not use unsupported options for transactions of parallel workers performing RestoreRelationTask

doc/sql.extensions/README.set_transaction.txt

@@ -27,6 +27,13 @@ commit fails. This option is mostly used by gfix.
 LOCK TIMEOUT nonneg_short_integer: it's the time (measured in seconds) that a
 transaction waits for a lock in a record before giving up and reporting an error.
 
+AUTO RELEASE TEMP BLOBID: makes the transaction release a temporary ID of a user
+BLOB just after its materialization. It's useful for massive insertions of records
+with user-defined BLOBs because it eliminates the memory overhead caused by creating
+and keeping temporary IDs until the transaction ends. This option should be used
+with care and only if there is no need to access a materialized BLOB via a temporary
+ID obtained after its creation. It's used during the database restore.
+
 
 Author:
    Claudio Valderrama C.

src/burp/BurpTasks.cpp

@@ -869,6 +869,8 @@ void RestoreRelationTask::initItem(BurpGlobals* tdgbl, Item& item)
 	tdgbl->verboseInterval = m_masterGbl->verboseInterval;
 	tdgbl->RESTORE_format = m_masterGbl->RESTORE_format;
 	tdgbl->runtimeODS = m_masterGbl->runtimeODS;
+	tdgbl->gbl_use_no_auto_undo = m_masterGbl->gbl_use_no_auto_undo;
+	tdgbl->gbl_use_auto_release_temp_blobid = m_masterGbl->gbl_use_auto_release_temp_blobid;
 
 	if (item.m_ownAttach)
 	{
@@ -893,11 +895,16 @@ void RestoreRelationTask::initItem(BurpGlobals* tdgbl, Item& item)
 			if (status->getState() & IStatus::STATE_ERRORS)
 				BURP_abort(&status);
 
-			// SET TRANSACTION NO_AUTO_UNDO, see at the end of get_data()
+			// SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID, see at the end of get_data()
 
 			ClumpletWriter tpb(ClumpletReader::Tpb, 128, isc_tpb_version3);
 			tpb.insertTag(isc_tpb_concurrency);
-			tpb.insertTag(isc_tpb_no_auto_undo);
+
+			if (tdgbl->gbl_use_no_auto_undo)
+				tpb.insertTag(isc_tpb_no_auto_undo);
+
+			if (tdgbl->gbl_use_auto_release_temp_blobid)
+				tpb.insertTag(isc_tpb_auto_release_temp_blobid);
 
 			item.m_tra = item.m_att->startTransaction(&status, tpb.getBufferLength(), tpb.getBuffer());
 

src/burp/burp.h

@@ -1227,6 +1227,9 @@ public:
 	bool gbl_stat_header;				// true, if stats header was printed
 	bool gbl_stat_done;					// true, if main process is done, stop to collect db-level stats
 	SINT64 gbl_stats[LAST_COUNTER];
+
+	bool gbl_use_no_auto_undo = true;
+	bool gbl_use_auto_release_temp_blobid = true;
 };
 
 // CVC: This aux routine declared here to not force inclusion of burp.h with burp_proto.h

src/burp/restore.epp

@@ -183,6 +183,7 @@ void	update_ownership(BurpGlobals* tdgbl);
 void	update_view_dbkey_lengths(BurpGlobals* tdgbl);
 void	fix_missing_privileges(BurpGlobals* tdgbl);
 void	fix_system_generators(BurpGlobals* tdgbl);
+void	set_transaction(BurpGlobals* tdgbl);
 void	general_on_error();
 #ifdef DEBUG
 UCHAR	debug_on = 0;   // able to turn this on in the debugger
@@ -745,9 +746,7 @@ void add_files(BurpGlobals* tdgbl, const char* file_name)
 			END_ERROR;
 		END_ERROR;
 
-		EXEC SQL SET TRANSACTION NO_AUTO_UNDO;
+		set_transaction(tdgbl);
-		if (gds_status->hasData())
-			EXEC SQL SET TRANSACTION;
 	}
 }
 
@@ -3166,9 +3165,7 @@ static void commit_relation_data(BurpGlobals* tdgbl, burp_rel* relation)
 		} // end of while
 	END_ERROR;
 
-	EXEC SQL SET TRANSACTION NO_AUTO_UNDO;
+	set_transaction(tdgbl);
-	if (gds_status->hasData())
-		EXEC SQL SET TRANSACTION;
 }
 
 // We have a corrupt backup, save the restore process from becoming useless.
@@ -7777,9 +7774,7 @@ bool get_relation(BurpGlobals* tdgbl, Coordinator* coord, RestoreRelationTask* t
 			general_on_error ();
 		END_ERROR;
 
-		EXEC SQL SET TRANSACTION NO_AUTO_UNDO;
+		set_transaction(tdgbl);
-		if (gds_status->hasData())
-			EXEC SQL SET TRANSACTION;
 	}
 
 	// Pick up relation attributes
@@ -8004,9 +7999,8 @@ bool get_relation(BurpGlobals* tdgbl, Coordinator* coord, RestoreRelationTask* t
 						general_on_error ();
 					END_ERROR;
 				END_ERROR;
-				EXEC SQL SET TRANSACTION NO_AUTO_UNDO;
+
-				if (gds_status->hasData())
+				set_transaction(tdgbl);
-					EXEC SQL SET TRANSACTION;
 			}
 			return true;
 
@@ -8041,9 +8035,7 @@ bool get_relation(BurpGlobals* tdgbl, Coordinator* coord, RestoreRelationTask* t
 		general_on_error ();
 	END_ERROR;
 
-	EXEC SQL SET TRANSACTION NO_AUTO_UNDO;
+	set_transaction(tdgbl);
-	if (gds_status->hasData())
-		EXEC SQL SET TRANSACTION;
 
 	// If we're only doing meta-data, ignore data records
 
@@ -9085,9 +9077,8 @@ bool get_trigger_old (BurpGlobals* tdgbl, burp_rel* relation)
 				general_on_error ();
 			END_ERROR;
 		END_ERROR;
-		EXEC SQL SET TRANSACTION NO_AUTO_UNDO;
+
-		if (gds_status->hasData())
+		set_transaction(tdgbl);
-			EXEC SQL SET TRANSACTION;
 	}
 
 	return true;
@@ -9441,9 +9432,8 @@ bool get_trigger(BurpGlobals* tdgbl)
 				general_on_error ();
 			END_ERROR;
 		END_ERROR;
-		EXEC SQL SET TRANSACTION NO_AUTO_UNDO;
+
-		if (gds_status->hasData())
+		set_transaction(tdgbl);
-			EXEC SQL SET TRANSACTION;
 	}
 
 	return true;
@@ -9537,9 +9527,8 @@ bool get_trigger_message(BurpGlobals* tdgbl)
 				general_on_error ();
 			END_ERROR;
 		END_ERROR;
-		EXEC SQL SET TRANSACTION NO_AUTO_UNDO;
+
-		if (gds_status->hasData())
+		set_transaction(tdgbl);
-			EXEC SQL SET TRANSACTION;
 	}
 
 	return true;
@@ -10391,9 +10380,7 @@ bool restore(BurpGlobals* tdgbl, Firebird::IProvider* provider, const TEXT* file
 
 	create_database(tdgbl, provider, database_name);
 
-	EXEC SQL SET TRANSACTION NO_AUTO_UNDO;
+	set_transaction(tdgbl);
-	if (gds_status->hasData())
-		EXEC SQL SET TRANSACTION;
 
 	// For V4.0, start a read commited transaction.  This will be used
 	// to create blobs for global fields and update the record in the
@@ -10751,9 +10738,8 @@ bool restore(BurpGlobals* tdgbl, Firebird::IProvider* provider, const TEXT* file
 				ON_ERROR
 					general_on_error ();
 				END_ERROR;
-				EXEC SQL SET TRANSACTION NO_AUTO_UNDO;
+
-				if (gds_status->hasData())
+				set_transaction(tdgbl);
-					EXEC SQL SET TRANSACTION;
 				flag = false;
 			}
 			if (!get_relation_data(tdgbl, &coord, &task))
@@ -10875,9 +10861,8 @@ bool restore(BurpGlobals* tdgbl, Firebird::IProvider* provider, const TEXT* file
 		ON_ERROR
 			general_on_error ();
 		END_ERROR;
-		EXEC SQL SET TRANSACTION NO_AUTO_UNDO;
+
-		if (gds_status->hasData())
+		set_transaction(tdgbl);
-			EXEC SQL SET TRANSACTION;
 	}
 
 	// put validation clauses for global fields
@@ -11625,6 +11610,52 @@ void fix_system_generators(BurpGlobals* tdgbl)
 	}
 }
 
+void set_transaction(BurpGlobals* tdgbl)
+{
+/**************************************
+ *
+ *	s e t _ t r a n s a c t i o n
+ *
+ **************************************
+ *
+ * Functional description
+ *	Start a transaction with options
+ *	supported by the target server.
+ *
+ **************************************/
+
+	while (true)
+	{
+		if (tdgbl->gbl_use_auto_release_temp_blobid && tdgbl->gbl_use_no_auto_undo)
+		{
+			EXEC SQL SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID;
+			if (gds_status->hasData() && fb_utils::containsErrorCode(gds_status->getErrors(), isc_bad_tpb_form))
+			{
+				// First try to disable AUTO_RELEASE_TEMP_BLOBID transaction
+				// option because it was implemented later than NO_AUTO_UNDO
+				tdgbl->gbl_use_auto_release_temp_blobid = false;
+				continue;
+			}
+		}
+		else if (tdgbl->gbl_use_no_auto_undo)
+		{
+			EXEC SQL SET TRANSACTION NO_AUTO_UNDO;
+			if (gds_status->hasData() && fb_utils::containsErrorCode(gds_status->getErrors(), isc_bad_tpb_form))
+			{
+				tdgbl->gbl_use_no_auto_undo = false;
+				continue;
+			}
+		}
+		else
+		{
+			fb_assert(!tdgbl->gbl_use_auto_release_temp_blobid);
+			EXEC SQL SET TRANSACTION;
+		}
+
+		break;
+	}
+}
+
 } // namespace
 
 namespace Burp

src/common/ParserTokens.h

@@ -97,6 +97,7 @@ PARSER_TOKEN(TOK_BINARY, "BINARY", false)
 PARSER_TOKEN(TOK_BIND, "BIND", true)
 PARSER_TOKEN(TOK_BIT_LENGTH, "BIT_LENGTH", false)
 PARSER_TOKEN(TOK_BLOB, "BLOB", false)
+PARSER_TOKEN(TOK_BLOBID, "BLOBID", true)
 PARSER_TOKEN(TOK_BLOB_APPEND, "BLOB_APPEND", true)
 PARSER_TOKEN(TOK_BLOCK, "BLOCK", true)
 PARSER_TOKEN(TOK_BODY, "BODY", true)
@@ -491,6 +492,7 @@ PARSER_TOKEN(TOK_TAGS, "TAGS", true)
 PARSER_TOKEN(TOK_TAN, "TAN", true)
 PARSER_TOKEN(TOK_TANH, "TANH", true)
 PARSER_TOKEN(TOK_TARGET, "TARGET", true)
+PARSER_TOKEN(TOK_TEMP, "TEMP", true)
 PARSER_TOKEN(TOK_TEMPORARY, "TEMPORARY", true)
 PARSER_TOKEN(TOK_THEN, "THEN", false)
 PARSER_TOKEN(TOK_TIES, "TIES", true)

src/dsql/StmtNodes.cpp

@@ -9720,6 +9720,9 @@ SetTransactionNode* SetTransactionNode::dsqlPass(DsqlCompilerScratch* dsqlScratc
 	if (autoCommit.isAssigned())
 		dsqlScratch->appendUChar(isc_tpb_autocommit);
 
+	if (autoReleaseTempBlobID.isAssigned())
+		dsqlScratch->appendUChar(isc_tpb_auto_release_temp_blobid);
+
 	if (lockTimeout.has_value())
 	{
 		dsqlScratch->appendUChar(isc_tpb_lock_timeout);

src/dsql/StmtNodes.h

@@ -1607,6 +1607,7 @@ public:
 		NODE_PRINT(printer, ignoreLimbo);
 		NODE_PRINT(printer, restartRequests);
 		NODE_PRINT(printer, autoCommit);
+		NODE_PRINT(printer, autoReleaseTempBlobID);
 		NODE_PRINT(printer, lockTimeout);
 		//// FIXME-PRINT: NODE_PRINT(printer, reserveList);
 		NODE_PRINT(printer, tpb);
@@ -1633,6 +1634,7 @@ public:
 	Firebird::TriState ignoreLimbo;
 	Firebird::TriState restartRequests;
 	Firebird::TriState autoCommit;
+	Firebird::TriState autoReleaseTempBlobID;
 };
 
 

src/dsql/parse.y

@@ -343,10 +343,12 @@ using namespace Firebird;
 
 %token <metaNamePtr> ACTION
 %token <metaNamePtr> ADMIN
+%token <metaNamePtr> BLOBID
 %token <metaNamePtr> CASCADE
 %token <metaNamePtr> FREE_IT			// ISC SQL extension
 %token <metaNamePtr> RESTRICT
 %token <metaNamePtr> ROLE
+%token <metaNamePtr> TEMP
 
 // New tokens added v6.0
 
@@ -5938,6 +5940,8 @@ tran_option($setTransactionNode)
 		{ setClause($setTransactionNode->restartRequests, "RESTART REQUESTS", true); }
 	| AUTO COMMIT
 		{ setClause($setTransactionNode->autoCommit, "AUTO COMMIT", true); }
+	| AUTO RELEASE TEMP BLOBID
+		{ setClause($setTransactionNode->autoReleaseTempBlobID, "AUTO RELEASE TEMP BLOBID", true); }
 	// timeout
 	| LOCK TIMEOUT nonneg_short_integer
 		{ setClause($setTransactionNode->lockTimeout, "LOCK TIMEOUT", (USHORT) $3); }
@@ -9749,10 +9753,12 @@ non_reserved_word
 	// added in FB 4.0.2
 	| BLOB_APPEND
 	// added in FB 5.0
+	| BLOBID
 	| LOCKED
 	| OPTIMIZE
 	| QUARTER
 	| TARGET
+	| TEMP
 	| TIMEZONE_NAME
 	| UNICODE_CHAR
 	| UNICODE_VAL

src/gpre/cmp.cpp

@@ -408,6 +408,8 @@ void CMP_t_start( gpre_tra* trans)
 		*text++ = isc_tpb_autocommit;
 	if (trans->tra_flags & TRA_no_auto_undo)
 		*text++ = isc_tpb_no_auto_undo;
+	if (trans->tra_flags & TRA_auto_release_temp_blobid)
+		*text++ = isc_tpb_auto_release_temp_blobid;
 	*text = 0;
 	const USHORT tpb_len = text - tpb_buffer;
 

src/gpre/gpre.h

@@ -1414,7 +1414,8 @@ enum tra_flags_vals {
 	TRA_read_committed	= 32,
 	TRA_autocommit		= 64,
 	TRA_rec_version		= 128,
-	TRA_no_auto_undo	= 256
+	TRA_no_auto_undo	= 256,
+	TRA_auto_release_temp_blobid = 512
 };
 
 const int MAX_TRA_OPTIONS	= 8;

src/gpre/hsh.h

@@ -43,6 +43,7 @@
 	{"AT", KW_AT},
 	{"AUTO", KW_AUTO},
 	{"AUTOCOMMIT", KW_AUTOCOMMIT},
+	{"AUTO_RELEASE_TEMP_BLOBID", KW_AUTO_RELEASE_TEMP_BLOBID},
 	{"AVERAGE", KW_AVERAGE},
 	{"AVG", KW_AVERAGE},
 	{"\\", KW_BACK_SLASH},

src/gpre/sql.cpp

@@ -4424,6 +4424,12 @@ static act* act_set_transaction()
 			continue;
 		}
 
+		if (MSC_match(KW_AUTO_RELEASE_TEMP_BLOBID))
+		{
+			trans->tra_flags |= TRA_auto_release_temp_blobid;
+			continue;
+		}
+
 		break;
 	}
 

src/gpre/words.h

@@ -113,6 +113,7 @@ enum kwwords_t {
 	KW_ASTERISK,
 	KW_AUTO,
 	KW_AUTOCOMMIT,
+	KW_AUTO_RELEASE_TEMP_BLOBID,
 	KW_AVERAGE,
 	KW_BASE_NAME,
 	KW_BETWEEN,

src/include/firebird/impl/consts_pub.h

@@ -268,6 +268,7 @@
 #define isc_tpb_lock_timeout              21
 #define isc_tpb_read_consistency          22
 #define isc_tpb_at_snapshot_number        23
+#define isc_tpb_auto_release_temp_blobid  24
 
 
 /************************/

src/include/gen/Firebird.pas

@@ -4111,6 +4111,7 @@ const
 	isc_tpb_lock_timeout = byte(21);
 	isc_tpb_read_consistency = byte(22);
 	isc_tpb_at_snapshot_number = byte(23);
+	isc_tpb_auto_release_temp_blobid = byte(24);
 	isc_bpb_version1 = byte(1);
 	isc_bpb_source_type = byte(1);
 	isc_bpb_target_type = byte(2);

src/jrd/Monitoring.cpp

@@ -1138,6 +1138,9 @@ void Monitoring::putTransaction(SnapshotData::DumpRecord& record, const jrd_tra*
 	// statistics
 	const int stat_id = fb_utils::genUniqueId();
 	record.storeGlobalId(f_mon_tra_stat_id, getGlobalId(stat_id));
+	// auto release temp blobid flag
+	temp = (transaction->tra_flags & TRA_auto_release_temp_blobid) ? 1 : 0;
+	record.storeInteger(f_mon_tra_auto_release_temp_blobid, temp);
 
 	record.write();
 

src/jrd/blb.cpp

@@ -326,6 +326,9 @@ blb* blb::create2(thread_db* tdbb,
 
 	blb* blob = allocate_blob(tdbb, transaction);
 
+	if (userBlob)
+		blob->blb_flags |= BLB_user;
+
 	if (type & isc_bpb_type_stream)
 		blob->blb_flags |= BLB_stream;
 
@@ -1296,7 +1299,9 @@ void blb::move(thread_db* tdbb, dsc* from_desc, dsc* to_desc,
 			array->arr_request = own_request;
 	}
 
-	blob->destroy(!materialized_blob);
+	const bool purgeBlob = !materialized_blob ||
+		((transaction->tra_flags & TRA_auto_release_temp_blobid) && (blob->blb_flags & BLB_user));
+	blob->destroy(purgeBlob);
 }
 
 

src/jrd/blb.h

@@ -181,6 +181,7 @@ const int BLB_seek			= 32;		// Seek is pending
 const int BLB_large_scan	= 64;		// Blob is larger than page buffer cache
 const int BLB_close_on_read = 128;		// Temporary blob is not closed until read
 const int BLB_bulk			= 256;		// Blob created by bulk insert operation
+const int BLB_user			= 512;		// User-defined blob
 
 /* Blob levels are:
 

src/jrd/names.h

@@ -287,6 +287,7 @@ NAME("MON$ATTACHMENT_ID", nam_mon_att_id)
 NAME("MON$ATTACHMENT_NAME", nam_mon_att_name)
 NAME("MON$AUTH_METHOD", nam_mon_auth_method)
 NAME("MON$AUTO_COMMIT", nam_mon_auto_commit)
+NAME("MON$AUTO_RELEASE_TEMP_BLOBID", nam_mon_auto_release_temp_blobid)
 NAME("MON$AUTO_UNDO", nam_mon_auto_undo)
 NAME("MON$BACKUP_STATE", nam_mon_backup_state)
 NAME("MON$BACKVERSION_READS", nam_mon_bkversion_reads)

src/jrd/relations.h

@@ -550,6 +550,7 @@ RELATION(nam_mon_transactions, rel_mon_transactions, ODS_11_1, rel_virtual)
 	FIELD(f_mon_tra_auto_commit, nam_mon_auto_commit, fld_flag_nullable, 0, ODS_11_1)
 	FIELD(f_mon_tra_auto_undo, nam_mon_auto_undo, fld_flag_nullable, 0, ODS_11_1)
 	FIELD(f_mon_tra_stat_id, nam_mon_stat_id, fld_stat_id, 0, ODS_11_1)
+	FIELD(f_mon_tra_auto_release_temp_blobid, nam_mon_auto_release_temp_blobid, fld_flag_nullable, 0, ODS_14_0)
 END_RELATION
 
 // Relation 36 (MON$STATEMENTS)

src/jrd/tra.cpp

@@ -3162,6 +3162,10 @@ static void transaction_options(thread_db* tdbb,
 			transaction->tra_flags |= TRA_no_auto_undo;
 			break;
 
+		case isc_tpb_auto_release_temp_blobid:
+			transaction->tra_flags |= TRA_auto_release_temp_blobid;
+			break;
+
 		case isc_tpb_lock_write:
 			// Cannot set a R/W table reservation if the whole txn is R/O.
 			if (read_only.asBool())

src/jrd/tra.h

@@ -435,10 +435,11 @@ const ULONG TRA_read_consistency	= 0x40000L; 	// ensure read consistency in this
 const ULONG TRA_ex_restart			= 0x80000L; 	// Exception was raised to restart request
 const ULONG TRA_replicating			= 0x100000L;	// transaction is allowed to be replicated
 const ULONG TRA_no_blob_check		= 0x200000L;	// disable blob access checking
+const ULONG TRA_auto_release_temp_blobid	= 0x400000L;	// remove temp ids of materialized user blobs from tra_blobs
 
 // flags derived from TPB, see also transaction_options() at tra.cpp
 const ULONG TRA_OPTIONS_MASK = (TRA_degree3 | TRA_readonly | TRA_ignore_limbo | TRA_read_committed |
-	TRA_autocommit | TRA_rec_version | TRA_read_consistency | TRA_no_auto_undo | TRA_restart_requests);
+	TRA_autocommit | TRA_rec_version | TRA_read_consistency | TRA_no_auto_undo | TRA_restart_requests | TRA_auto_release_temp_blobid);
 
 const int TRA_MASK				= 3;
 //const int TRA_BITS_PER_TRANS	= 2;