firebird/firebird-mirror
firebird/firebird-mirror
8
0
Fork 1
mirror of https://github.com/FirebirdSQL/firebird.git synced 2025-01-22 18:03:03 +01:00
Code Issues

Add AUTO RELEASE TEMP BLOBID transaction option (#8323)

Browse Source 
* Add AUTO RELEASE TEMP BLOBID transaction option

It makes the transaction release temporary ID of user BLOB just after its materialization. It's useful for massive insertions of records with user-defined BLOBs because it eliminates the memory overhead caused by creating and keeping temporary IDs until the transaction ends. This option is used during the database restore.

* Place tokens in the correct sections

* Avoid repeated attempts to start a transaction with options that are not supported by the target server

* Correct AUTO RELEASE TEMP BLOBID description

* Check bad_tpb_form error for more reliable detection of unsupported transaction options

* Do not use unsupported options for transactions of parallel workers performing RestoreRelationTask
This commit is contained in:
Ilya Eremin 2024-11-25 11:32:33 +03:00 committed by GitHub
parent 933ac7bf19
commit babd741daa
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
22 changed files with 128 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
doc/sql.extensions/README.set_transaction.txt
View File

@ -27,6 +27,13 @@ commit fails. This option is mostly used by gfix.
LOCK TIMEOUT nonneg_short_integer: it's the time (measured in seconds) that a
transaction waits for a lock in a record before giving up and reporting an error.
AUTO RELEASE TEMP BLOBID: makes the transaction release a temporary ID of a user
BLOB just after its materialization. It's useful for massive insertions of records
with user-defined BLOBs because it eliminates the memory overhead caused by creating
and keeping temporary IDs until the transaction ends. This option should be used
with care and only if there is no need to access a materialized BLOB via a temporary
ID obtained after its creation. It's used during the database restore.
Author:
Claudio Valderrama C.

9
src/burp/BurpTasks.cpp
View File

@ -869,6 +869,8 @@ void RestoreRelationTask::initItem(BurpGlobals* tdgbl, Item& item)
tdgbl->verboseInterval = m_masterGbl->verboseInterval;
tdgbl->RESTORE_format = m_masterGbl->RESTORE_format;
tdgbl->runtimeODS = m_masterGbl->runtimeODS;
tdgbl->gbl_use_no_auto_undo = m_masterGbl->gbl_use_no_auto_undo;
tdgbl->gbl_use_auto_release_temp_blobid = m_masterGbl->gbl_use_auto_release_temp_blobid;
if (item.m_ownAttach)
{
@ -893,12 +895,17 @@ void RestoreRelationTask::initItem(BurpGlobals* tdgbl, Item& item)
if (status->getState() & IStatus::STATE_ERRORS)
BURP_abort(&status);
// SET TRANSACTION NO_AUTO_UNDO, see at the end of get_data()
// SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID, see at the end of get_data()
ClumpletWriter tpb(ClumpletReader::Tpb, 128, isc_tpb_version3);
tpb.insertTag(isc_tpb_concurrency);
if (tdgbl->gbl_use_no_auto_undo)
tpb.insertTag(isc_tpb_no_auto_undo);
if (tdgbl->gbl_use_auto_release_temp_blobid)
tpb.insertTag(isc_tpb_auto_release_temp_blobid);
item.m_tra = item.m_att->startTransaction(&status, tpb.getBufferLength(), tpb.getBuffer());
if (status->getState() & IStatus::STATE_ERRORS)

3
src/burp/burp.h
View File

@ -1227,6 +1227,9 @@ public:
bool gbl_stat_header; // true, if stats header was printed
bool gbl_stat_done; // true, if main process is done, stop to collect db-level stats
SINT64 gbl_stats[LAST_COUNTER];
bool gbl_use_no_auto_undo = true;
bool gbl_use_auto_release_temp_blobid = true;
};
// CVC: This aux routine declared here to not force inclusion of burp.h with burp_proto.h

97
src/burp/restore.epp
View File

@ -183,6 +183,7 @@ void update_ownership(BurpGlobals* tdgbl);
void update_view_dbkey_lengths(BurpGlobals* tdgbl);
void fix_missing_privileges(BurpGlobals* tdgbl);
void fix_system_generators(BurpGlobals* tdgbl);
void set_transaction(BurpGlobals* tdgbl);
void general_on_error();
#ifdef DEBUG
UCHAR debug_on = 0; // able to turn this on in the debugger
@ -745,9 +746,7 @@ void add_files(BurpGlobals* tdgbl, const char* file_name)
END_ERROR;
END_ERROR;
EXEC SQL SET TRANSACTION NO_AUTO_UNDO;
if (gds_status->hasData())
EXEC SQL SET TRANSACTION;
set_transaction(tdgbl);
}
}
@ -3166,9 +3165,7 @@ static void commit_relation_data(BurpGlobals* tdgbl, burp_rel* relation)
} // end of while
END_ERROR;
EXEC SQL SET TRANSACTION NO_AUTO_UNDO;
if (gds_status->hasData())
EXEC SQL SET TRANSACTION;
set_transaction(tdgbl);
}
// We have a corrupt backup, save the restore process from becoming useless.
@ -7777,9 +7774,7 @@ bool get_relation(BurpGlobals* tdgbl, Coordinator* coord, RestoreRelationTask* t
general_on_error ();
END_ERROR;
EXEC SQL SET TRANSACTION NO_AUTO_UNDO;
if (gds_status->hasData())
EXEC SQL SET TRANSACTION;
set_transaction(tdgbl);
}
// Pick up relation attributes
@ -8004,9 +7999,8 @@ bool get_relation(BurpGlobals* tdgbl, Coordinator* coord, RestoreRelationTask* t
general_on_error ();
END_ERROR;
END_ERROR;
EXEC SQL SET TRANSACTION NO_AUTO_UNDO;
if (gds_status->hasData())
EXEC SQL SET TRANSACTION;
set_transaction(tdgbl);
}
return true;
@ -8041,9 +8035,7 @@ bool get_relation(BurpGlobals* tdgbl, Coordinator* coord, RestoreRelationTask* t
general_on_error ();
END_ERROR;
EXEC SQL SET TRANSACTION NO_AUTO_UNDO;
if (gds_status->hasData())
EXEC SQL SET TRANSACTION;
set_transaction(tdgbl);
// If we're only doing meta-data, ignore data records
@ -9085,9 +9077,8 @@ bool get_trigger_old (BurpGlobals* tdgbl, burp_rel* relation)
general_on_error ();
END_ERROR;
END_ERROR;
EXEC SQL SET TRANSACTION NO_AUTO_UNDO;
if (gds_status->hasData())
EXEC SQL SET TRANSACTION;
set_transaction(tdgbl);
}
return true;
@ -9441,9 +9432,8 @@ bool get_trigger(BurpGlobals* tdgbl)
general_on_error ();
END_ERROR;
END_ERROR;
EXEC SQL SET TRANSACTION NO_AUTO_UNDO;
if (gds_status->hasData())
EXEC SQL SET TRANSACTION;
set_transaction(tdgbl);
}
return true;
@ -9537,9 +9527,8 @@ bool get_trigger_message(BurpGlobals* tdgbl)
general_on_error ();
END_ERROR;
END_ERROR;
EXEC SQL SET TRANSACTION NO_AUTO_UNDO;
if (gds_status->hasData())
EXEC SQL SET TRANSACTION;
set_transaction(tdgbl);
}
return true;
@ -10391,9 +10380,7 @@ bool restore(BurpGlobals* tdgbl, Firebird::IProvider* provider, const TEXT* file
create_database(tdgbl, provider, database_name);
EXEC SQL SET TRANSACTION NO_AUTO_UNDO;
if (gds_status->hasData())
EXEC SQL SET TRANSACTION;
set_transaction(tdgbl);
// For V4.0, start a read commited transaction. This will be used
// to create blobs for global fields and update the record in the
@ -10751,9 +10738,8 @@ bool restore(BurpGlobals* tdgbl, Firebird::IProvider* provider, const TEXT* file
ON_ERROR
general_on_error ();
END_ERROR;
EXEC SQL SET TRANSACTION NO_AUTO_UNDO;
if (gds_status->hasData())
EXEC SQL SET TRANSACTION;
set_transaction(tdgbl);
flag = false;
}
if (!get_relation_data(tdgbl, &coord, &task))
@ -10875,9 +10861,8 @@ bool restore(BurpGlobals* tdgbl, Firebird::IProvider* provider, const TEXT* file
ON_ERROR
general_on_error ();
END_ERROR;
EXEC SQL SET TRANSACTION NO_AUTO_UNDO;
if (gds_status->hasData())
EXEC SQL SET TRANSACTION;
set_transaction(tdgbl);
}
// put validation clauses for global fields
@ -11625,6 +11610,52 @@ void fix_system_generators(BurpGlobals* tdgbl)
}
}
void set_transaction(BurpGlobals* tdgbl)
{
/**************************************
*
* s e t _ t r a n s a c t i o n
*
**************************************
*
* Functional description
* Start a transaction with options
* supported by the target server.
*
**************************************/
while (true)
{
if (tdgbl->gbl_use_auto_release_temp_blobid && tdgbl->gbl_use_no_auto_undo)
{
EXEC SQL SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID;
if (gds_status->hasData() && fb_utils::containsErrorCode(gds_status->getErrors(), isc_bad_tpb_form))
{
// First try to disable AUTO_RELEASE_TEMP_BLOBID transaction
// option because it was implemented later than NO_AUTO_UNDO
tdgbl->gbl_use_auto_release_temp_blobid = false;
continue;
}
}
else if (tdgbl->gbl_use_no_auto_undo)
{
EXEC SQL SET TRANSACTION NO_AUTO_UNDO;
if (gds_status->hasData() && fb_utils::containsErrorCode(gds_status->getErrors(), isc_bad_tpb_form))
{
tdgbl->gbl_use_no_auto_undo = false;
continue;
}
}
else
{
fb_assert(!tdgbl->gbl_use_auto_release_temp_blobid);
EXEC SQL SET TRANSACTION;
}
break;
}
}
} // namespace
namespace Burp

2
src/common/ParserTokens.h
View File

@ -97,6 +97,7 @@ PARSER_TOKEN(TOK_BINARY, "BINARY", false)
PARSER_TOKEN(TOK_BIND, "BIND", true)
PARSER_TOKEN(TOK_BIT_LENGTH, "BIT_LENGTH", false)
PARSER_TOKEN(TOK_BLOB, "BLOB", false)
PARSER_TOKEN(TOK_BLOBID, "BLOBID", true)
PARSER_TOKEN(TOK_BLOB_APPEND, "BLOB_APPEND", true)
PARSER_TOKEN(TOK_BLOCK, "BLOCK", true)
PARSER_TOKEN(TOK_BODY, "BODY", true)
@ -491,6 +492,7 @@ PARSER_TOKEN(TOK_TAGS, "TAGS", true)
PARSER_TOKEN(TOK_TAN, "TAN", true)
PARSER_TOKEN(TOK_TANH, "TANH", true)
PARSER_TOKEN(TOK_TARGET, "TARGET", true)
PARSER_TOKEN(TOK_TEMP, "TEMP", true)
PARSER_TOKEN(TOK_TEMPORARY, "TEMPORARY", true)
PARSER_TOKEN(TOK_THEN, "THEN", false)
PARSER_TOKEN(TOK_TIES, "TIES", true)

3
src/dsql/StmtNodes.cpp
View File

@ -9720,6 +9720,9 @@ SetTransactionNode* SetTransactionNode::dsqlPass(DsqlCompilerScratch* dsqlScratc
if (autoCommit.isAssigned())
dsqlScratch->appendUChar(isc_tpb_autocommit);
if (autoReleaseTempBlobID.isAssigned())
dsqlScratch->appendUChar(isc_tpb_auto_release_temp_blobid);
if (lockTimeout.has_value())
{
dsqlScratch->appendUChar(isc_tpb_lock_timeout);

2
src/dsql/StmtNodes.h
View File

@ -1607,6 +1607,7 @@ public:
NODE_PRINT(printer, ignoreLimbo);
NODE_PRINT(printer, restartRequests);
NODE_PRINT(printer, autoCommit);
NODE_PRINT(printer, autoReleaseTempBlobID);
NODE_PRINT(printer, lockTimeout);
//// FIXME-PRINT: NODE_PRINT(printer, reserveList);
NODE_PRINT(printer, tpb);
@ -1633,6 +1634,7 @@ public:
Firebird::TriState ignoreLimbo;
Firebird::TriState restartRequests;
Firebird::TriState autoCommit;
Firebird::TriState autoReleaseTempBlobID;
};

6
src/dsql/parse.y
View File

@ -343,10 +343,12 @@ using namespace Firebird;
%token <metaNamePtr> ACTION
%token <metaNamePtr> ADMIN
%token <metaNamePtr> BLOBID
%token <metaNamePtr> CASCADE
%token <metaNamePtr> FREE_IT // ISC SQL extension
%token <metaNamePtr> RESTRICT
%token <metaNamePtr> ROLE
%token <metaNamePtr> TEMP
// New tokens added v6.0
@ -5938,6 +5940,8 @@ tran_option($setTransactionNode)
{ setClause($setTransactionNode->restartRequests, "RESTART REQUESTS", true); }
| AUTO COMMIT
{ setClause($setTransactionNode->autoCommit, "AUTO COMMIT", true); }
| AUTO RELEASE TEMP BLOBID
{ setClause($setTransactionNode->autoReleaseTempBlobID, "AUTO RELEASE TEMP BLOBID", true); }
// timeout
| LOCK TIMEOUT nonneg_short_integer
{ setClause($setTransactionNode->lockTimeout, "LOCK TIMEOUT", (USHORT) $3); }
@ -9749,10 +9753,12 @@ non_reserved_word
// added in FB 4.0.2
| BLOB_APPEND
// added in FB 5.0
| BLOBID
| LOCKED
| OPTIMIZE
| QUARTER
| TARGET
| TEMP
| TIMEZONE_NAME
| UNICODE_CHAR
| UNICODE_VAL

2
src/gpre/cmp.cpp
View File

@ -408,6 +408,8 @@ void CMP_t_start( gpre_tra* trans)
*text++ = isc_tpb_autocommit;
if (trans->tra_flags & TRA_no_auto_undo)
*text++ = isc_tpb_no_auto_undo;
if (trans->tra_flags & TRA_auto_release_temp_blobid)
*text++ = isc_tpb_auto_release_temp_blobid;
*text = 0;
const USHORT tpb_len = text - tpb_buffer;

3
src/gpre/gpre.h
View File

@ -1414,7 +1414,8 @@ enum tra_flags_vals {
TRA_read_committed = 32,
TRA_autocommit = 64,
TRA_rec_version = 128,
TRA_no_auto_undo = 256
TRA_no_auto_undo = 256,
TRA_auto_release_temp_blobid = 512
};
const int MAX_TRA_OPTIONS = 8;

1
src/gpre/hsh.h
View File

@ -43,6 +43,7 @@
{"AT", KW_AT},
{"AUTO", KW_AUTO},
{"AUTOCOMMIT", KW_AUTOCOMMIT},
{"AUTO_RELEASE_TEMP_BLOBID", KW_AUTO_RELEASE_TEMP_BLOBID},
{"AVERAGE", KW_AVERAGE},
{"AVG", KW_AVERAGE},
{"\\", KW_BACK_SLASH},

6
src/gpre/sql.cpp
View File

@ -4424,6 +4424,12 @@ static act* act_set_transaction()
continue;
}
if (MSC_match(KW_AUTO_RELEASE_TEMP_BLOBID))
{
trans->tra_flags |= TRA_auto_release_temp_blobid;
continue;
}
break;
}

1
src/gpre/words.h
View File

@ -113,6 +113,7 @@ enum kwwords_t {
KW_ASTERISK,
KW_AUTO,
KW_AUTOCOMMIT,
KW_AUTO_RELEASE_TEMP_BLOBID,
KW_AVERAGE,
KW_BASE_NAME,
KW_BETWEEN,

1
src/include/firebird/impl/consts_pub.h
View File

@ -268,6 +268,7 @@
#define isc_tpb_lock_timeout 21
#define isc_tpb_read_consistency 22
#define isc_tpb_at_snapshot_number 23
#define isc_tpb_auto_release_temp_blobid 24
/************************/

1
src/include/gen/Firebird.pas
View File

@ -4111,6 +4111,7 @@ const
isc_tpb_lock_timeout = byte(21);
isc_tpb_read_consistency = byte(22);
isc_tpb_at_snapshot_number = byte(23);
isc_tpb_auto_release_temp_blobid = byte(24);
isc_bpb_version1 = byte(1);
isc_bpb_source_type = byte(1);
isc_bpb_target_type = byte(2);

3
src/jrd/Monitoring.cpp
View File

@ -1138,6 +1138,9 @@ void Monitoring::putTransaction(SnapshotData::DumpRecord& record, const jrd_tra*
// statistics
const int stat_id = fb_utils::genUniqueId();
record.storeGlobalId(f_mon_tra_stat_id, getGlobalId(stat_id));
// auto release temp blobid flag
temp = (transaction->tra_flags & TRA_auto_release_temp_blobid) ? 1 : 0;
record.storeInteger(f_mon_tra_auto_release_temp_blobid, temp);
record.write();

7
src/jrd/blb.cpp
View File

@ -326,6 +326,9 @@ blb* blb::create2(thread_db* tdbb,
blb* blob = allocate_blob(tdbb, transaction);
if (userBlob)
blob->blb_flags |= BLB_user;
if (type & isc_bpb_type_stream)
blob->blb_flags |= BLB_stream;
@ -1296,7 +1299,9 @@ void blb::move(thread_db* tdbb, dsc* from_desc, dsc* to_desc,
array->arr_request = own_request;
}
blob->destroy(!materialized_blob);
const bool purgeBlob = !materialized_blob ||
((transaction->tra_flags & TRA_auto_release_temp_blobid) && (blob->blb_flags & BLB_user));
blob->destroy(purgeBlob);
}

1
src/jrd/blb.h
View File

@ -181,6 +181,7 @@ const int BLB_seek = 32; // Seek is pending
const int BLB_large_scan = 64; // Blob is larger than page buffer cache
const int BLB_close_on_read = 128; // Temporary blob is not closed until read
const int BLB_bulk = 256; // Blob created by bulk insert operation
const int BLB_user = 512; // User-defined blob
/* Blob levels are:

1
src/jrd/names.h
View File

@ -287,6 +287,7 @@ NAME("MON$ATTACHMENT_ID", nam_mon_att_id)
NAME("MON$ATTACHMENT_NAME", nam_mon_att_name)
NAME("MON$AUTH_METHOD", nam_mon_auth_method)
NAME("MON$AUTO_COMMIT", nam_mon_auto_commit)
NAME("MON$AUTO_RELEASE_TEMP_BLOBID", nam_mon_auto_release_temp_blobid)
NAME("MON$AUTO_UNDO", nam_mon_auto_undo)
NAME("MON$BACKUP_STATE", nam_mon_backup_state)
NAME("MON$BACKVERSION_READS", nam_mon_bkversion_reads)

1
src/jrd/relations.h
View File

@ -550,6 +550,7 @@ RELATION(nam_mon_transactions, rel_mon_transactions, ODS_11_1, rel_virtual)
FIELD(f_mon_tra_auto_commit, nam_mon_auto_commit, fld_flag_nullable, 0, ODS_11_1)
FIELD(f_mon_tra_auto_undo, nam_mon_auto_undo, fld_flag_nullable, 0, ODS_11_1)
FIELD(f_mon_tra_stat_id, nam_mon_stat_id, fld_stat_id, 0, ODS_11_1)
FIELD(f_mon_tra_auto_release_temp_blobid, nam_mon_auto_release_temp_blobid, fld_flag_nullable, 0, ODS_14_0)
END_RELATION
// Relation 36 (MON$STATEMENTS)

4
src/jrd/tra.cpp
View File

@ -3162,6 +3162,10 @@ static void transaction_options(thread_db* tdbb,
transaction->tra_flags |= TRA_no_auto_undo;
break;
case isc_tpb_auto_release_temp_blobid:
transaction->tra_flags |= TRA_auto_release_temp_blobid;
break;
case isc_tpb_lock_write:
// Cannot set a R/W table reservation if the whole txn is R/O.
if (read_only.asBool())

3
src/jrd/tra.h
View File

@ -435,10 +435,11 @@ const ULONG TRA_read_consistency = 0x40000L; // ensure read consistency in this
const ULONG TRA_ex_restart = 0x80000L; // Exception was raised to restart request
const ULONG TRA_replicating = 0x100000L; // transaction is allowed to be replicated
const ULONG TRA_no_blob_check = 0x200000L; // disable blob access checking
const ULONG TRA_auto_release_temp_blobid = 0x400000L; // remove temp ids of materialized user blobs from tra_blobs
// flags derived from TPB, see also transaction_options() at tra.cpp
const ULONG TRA_OPTIONS_MASK = (TRA_degree3 | TRA_readonly | TRA_ignore_limbo | TRA_read_committed |
TRA_autocommit | TRA_rec_version | TRA_read_consistency | TRA_no_auto_undo | TRA_restart_requests);
TRA_autocommit | TRA_rec_version | TRA_read_consistency | TRA_no_auto_undo | TRA_restart_requests | TRA_auto_release_temp_blobid);
const int TRA_MASK = 3;
//const int TRA_BITS_PER_TRANS = 2;