From df51372745004fbde7588d0606798046815927f5 Mon Sep 17 00:00:00 2001 From: Ilya Eremin Date: Mon, 25 Nov 2024 11:32:33 +0300 Subject: [PATCH] Backport from master: Add AUTO RELEASE TEMP BLOBID transaction option (#8323) * Add AUTO RELEASE TEMP BLOBID transaction option It makes the transaction release temporary ID of user BLOB just after its materialization. It's useful for massive insertions of records with user-defined BLOBs because it eliminates the memory overhead caused by creating and keeping temporary IDs until the transaction ends. This option is used during the database restore. * Place tokens in the correct sections * Avoid repeated attempts to start a transaction with options that are not supported by the target server * Correct AUTO RELEASE TEMP BLOBID description * Check bad_tpb_form error for more reliable detection of unsupported transaction options * Do not use unsupported options for transactions of parallel workers performing RestoreRelationTask --- doc/sql.extensions/README.set_transaction.txt | 7 ++ src/burp/BurpTasks.cpp | 11 ++- src/burp/burp.h | 3 + src/burp/restore.epp | 97 ++++++++++++------- src/common/ParserTokens.h | 2 + src/dsql/StmtNodes.cpp | 3 + src/dsql/StmtNodes.h | 2 + src/dsql/parse.y | 6 ++ src/gpre/cmp.cpp | 2 + src/gpre/gpre.h | 3 +- src/gpre/hsh.h | 1 + src/gpre/sql.cpp | 6 ++ src/gpre/words.h | 1 + src/include/firebird/impl/consts_pub.h | 1 + src/include/gen/Firebird.pas | 1 + src/jrd/blb.cpp | 7 +- src/jrd/blb.h | 1 + src/jrd/tra.cpp | 4 + src/jrd/tra.h | 3 +- 19 files changed, 123 insertions(+), 38 deletions(-) diff --git a/doc/sql.extensions/README.set_transaction.txt b/doc/sql.extensions/README.set_transaction.txt index 48e6463cb2..2b086167c4 100644 --- a/doc/sql.extensions/README.set_transaction.txt +++ b/doc/sql.extensions/README.set_transaction.txt @@ -27,6 +27,13 @@ commit fails. This option is mostly used by gfix. LOCK TIMEOUT nonneg_short_integer: it's the time (measured in seconds) that a transaction waits for a lock in a record before giving up and reporting an error. +AUTO RELEASE TEMP BLOBID: makes the transaction release a temporary ID of a user +BLOB just after its materialization. It's useful for massive insertions of records +with user-defined BLOBs because it eliminates the memory overhead caused by creating +and keeping temporary IDs until the transaction ends. This option should be used +with care and only if there is no need to access a materialized BLOB via a temporary +ID obtained after its creation. It's used during the database restore. + Author: Claudio Valderrama C. diff --git a/src/burp/BurpTasks.cpp b/src/burp/BurpTasks.cpp index c3ed549735..0f6411f114 100644 --- a/src/burp/BurpTasks.cpp +++ b/src/burp/BurpTasks.cpp @@ -869,6 +869,8 @@ void RestoreRelationTask::initItem(BurpGlobals* tdgbl, Item& item) tdgbl->verboseInterval = m_masterGbl->verboseInterval; tdgbl->RESTORE_format = m_masterGbl->RESTORE_format; tdgbl->runtimeODS = m_masterGbl->runtimeODS; + tdgbl->gbl_use_no_auto_undo = m_masterGbl->gbl_use_no_auto_undo; + tdgbl->gbl_use_auto_release_temp_blobid = m_masterGbl->gbl_use_auto_release_temp_blobid; if (item.m_ownAttach) { @@ -893,11 +895,16 @@ void RestoreRelationTask::initItem(BurpGlobals* tdgbl, Item& item) if (status->getState() & IStatus::STATE_ERRORS) BURP_abort(&status); - // SET TRANSACTION NO_AUTO_UNDO, see at the end of get_data() + // SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID, see at the end of get_data() ClumpletWriter tpb(ClumpletReader::Tpb, 128, isc_tpb_version3); tpb.insertTag(isc_tpb_concurrency); - tpb.insertTag(isc_tpb_no_auto_undo); + + if (tdgbl->gbl_use_no_auto_undo) + tpb.insertTag(isc_tpb_no_auto_undo); + + if (tdgbl->gbl_use_auto_release_temp_blobid) + tpb.insertTag(isc_tpb_auto_release_temp_blobid); item.m_tra = item.m_att->startTransaction(&status, tpb.getBufferLength(), tpb.getBuffer()); diff --git a/src/burp/burp.h b/src/burp/burp.h index be97f6dc4b..b920b32d09 100644 --- a/src/burp/burp.h +++ b/src/burp/burp.h @@ -1227,6 +1227,9 @@ public: bool gbl_stat_header; // true, if stats header was printed bool gbl_stat_done; // true, if main process is done, stop to collect db-level stats SINT64 gbl_stats[LAST_COUNTER]; + + bool gbl_use_no_auto_undo = true; + bool gbl_use_auto_release_temp_blobid = true; }; // CVC: This aux routine declared here to not force inclusion of burp.h with burp_proto.h diff --git a/src/burp/restore.epp b/src/burp/restore.epp index 90e57f32a4..b64506f046 100644 --- a/src/burp/restore.epp +++ b/src/burp/restore.epp @@ -182,6 +182,7 @@ void update_ownership(BurpGlobals* tdgbl); void update_view_dbkey_lengths(BurpGlobals* tdgbl); void fix_missing_privileges(BurpGlobals* tdgbl); void fix_system_generators(BurpGlobals* tdgbl); +void set_transaction(BurpGlobals* tdgbl); void general_on_error(); #ifdef DEBUG UCHAR debug_on = 0; // able to turn this on in the debugger @@ -744,9 +745,7 @@ void add_files(BurpGlobals* tdgbl, const char* file_name) END_ERROR; END_ERROR; - EXEC SQL SET TRANSACTION NO_AUTO_UNDO; - if (gds_status->hasData()) - EXEC SQL SET TRANSACTION; + set_transaction(tdgbl); } } @@ -3165,9 +3164,7 @@ static void commit_relation_data(BurpGlobals* tdgbl, burp_rel* relation) } // end of while END_ERROR; - EXEC SQL SET TRANSACTION NO_AUTO_UNDO; - if (gds_status->hasData()) - EXEC SQL SET TRANSACTION; + set_transaction(tdgbl); } // We have a corrupt backup, save the restore process from becoming useless. @@ -7776,9 +7773,7 @@ bool get_relation(BurpGlobals* tdgbl, Coordinator* coord, RestoreRelationTask* t general_on_error (); END_ERROR; - EXEC SQL SET TRANSACTION NO_AUTO_UNDO; - if (gds_status->hasData()) - EXEC SQL SET TRANSACTION; + set_transaction(tdgbl); } // Pick up relation attributes @@ -8003,9 +7998,8 @@ bool get_relation(BurpGlobals* tdgbl, Coordinator* coord, RestoreRelationTask* t general_on_error (); END_ERROR; END_ERROR; - EXEC SQL SET TRANSACTION NO_AUTO_UNDO; - if (gds_status->hasData()) - EXEC SQL SET TRANSACTION; + + set_transaction(tdgbl); } return true; @@ -8040,9 +8034,7 @@ bool get_relation(BurpGlobals* tdgbl, Coordinator* coord, RestoreRelationTask* t general_on_error (); END_ERROR; - EXEC SQL SET TRANSACTION NO_AUTO_UNDO; - if (gds_status->hasData()) - EXEC SQL SET TRANSACTION; + set_transaction(tdgbl); // If we're only doing meta-data, ignore data records @@ -9084,9 +9076,8 @@ bool get_trigger_old (BurpGlobals* tdgbl, burp_rel* relation) general_on_error (); END_ERROR; END_ERROR; - EXEC SQL SET TRANSACTION NO_AUTO_UNDO; - if (gds_status->hasData()) - EXEC SQL SET TRANSACTION; + + set_transaction(tdgbl); } return true; @@ -9440,9 +9431,8 @@ bool get_trigger(BurpGlobals* tdgbl) general_on_error (); END_ERROR; END_ERROR; - EXEC SQL SET TRANSACTION NO_AUTO_UNDO; - if (gds_status->hasData()) - EXEC SQL SET TRANSACTION; + + set_transaction(tdgbl); } return true; @@ -9536,9 +9526,8 @@ bool get_trigger_message(BurpGlobals* tdgbl) general_on_error (); END_ERROR; END_ERROR; - EXEC SQL SET TRANSACTION NO_AUTO_UNDO; - if (gds_status->hasData()) - EXEC SQL SET TRANSACTION; + + set_transaction(tdgbl); } return true; @@ -10390,9 +10379,7 @@ bool restore(BurpGlobals* tdgbl, Firebird::IProvider* provider, const TEXT* file create_database(tdgbl, provider, database_name); - EXEC SQL SET TRANSACTION NO_AUTO_UNDO; - if (gds_status->hasData()) - EXEC SQL SET TRANSACTION; + set_transaction(tdgbl); // For V4.0, start a read commited transaction. This will be used // to create blobs for global fields and update the record in the @@ -10750,9 +10737,8 @@ bool restore(BurpGlobals* tdgbl, Firebird::IProvider* provider, const TEXT* file ON_ERROR general_on_error (); END_ERROR; - EXEC SQL SET TRANSACTION NO_AUTO_UNDO; - if (gds_status->hasData()) - EXEC SQL SET TRANSACTION; + + set_transaction(tdgbl); flag = false; } if (!get_relation_data(tdgbl, &coord, &task)) @@ -10874,9 +10860,8 @@ bool restore(BurpGlobals* tdgbl, Firebird::IProvider* provider, const TEXT* file ON_ERROR general_on_error (); END_ERROR; - EXEC SQL SET TRANSACTION NO_AUTO_UNDO; - if (gds_status->hasData()) - EXEC SQL SET TRANSACTION; + + set_transaction(tdgbl); } // put validation clauses for global fields @@ -11624,6 +11609,52 @@ void fix_system_generators(BurpGlobals* tdgbl) } } +void set_transaction(BurpGlobals* tdgbl) +{ +/************************************** + * + * s e t _ t r a n s a c t i o n + * + ************************************** + * + * Functional description + * Start a transaction with options + * supported by the target server. + * + **************************************/ + + while (true) + { + if (tdgbl->gbl_use_auto_release_temp_blobid && tdgbl->gbl_use_no_auto_undo) + { + EXEC SQL SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID; + if (gds_status->hasData() && fb_utils::containsErrorCode(gds_status->getErrors(), isc_bad_tpb_form)) + { + // First try to disable AUTO_RELEASE_TEMP_BLOBID transaction + // option because it was implemented later than NO_AUTO_UNDO + tdgbl->gbl_use_auto_release_temp_blobid = false; + continue; + } + } + else if (tdgbl->gbl_use_no_auto_undo) + { + EXEC SQL SET TRANSACTION NO_AUTO_UNDO; + if (gds_status->hasData() && fb_utils::containsErrorCode(gds_status->getErrors(), isc_bad_tpb_form)) + { + tdgbl->gbl_use_no_auto_undo = false; + continue; + } + } + else + { + fb_assert(!tdgbl->gbl_use_auto_release_temp_blobid); + EXEC SQL SET TRANSACTION; + } + + break; + } +} + } // namespace namespace Burp diff --git a/src/common/ParserTokens.h b/src/common/ParserTokens.h index 29fb135b66..8f27da8514 100644 --- a/src/common/ParserTokens.h +++ b/src/common/ParserTokens.h @@ -95,6 +95,7 @@ PARSER_TOKEN(TOK_BINARY, "BINARY", false) PARSER_TOKEN(TOK_BIND, "BIND", true) PARSER_TOKEN(TOK_BIT_LENGTH, "BIT_LENGTH", false) PARSER_TOKEN(TOK_BLOB, "BLOB", false) +PARSER_TOKEN(TOK_BLOBID, "BLOBID", true) PARSER_TOKEN(TOK_BLOB_APPEND, "BLOB_APPEND", true) PARSER_TOKEN(TOK_BLOCK, "BLOCK", true) PARSER_TOKEN(TOK_BODY, "BODY", true) @@ -486,6 +487,7 @@ PARSER_TOKEN(TOK_TAGS, "TAGS", true) PARSER_TOKEN(TOK_TAN, "TAN", true) PARSER_TOKEN(TOK_TANH, "TANH", true) PARSER_TOKEN(TOK_TARGET, "TARGET", true) +PARSER_TOKEN(TOK_TEMP, "TEMP", true) PARSER_TOKEN(TOK_TEMPORARY, "TEMPORARY", true) PARSER_TOKEN(TOK_THEN, "THEN", false) PARSER_TOKEN(TOK_TIES, "TIES", true) diff --git a/src/dsql/StmtNodes.cpp b/src/dsql/StmtNodes.cpp index 0d1315f569..fd4b24b8b0 100644 --- a/src/dsql/StmtNodes.cpp +++ b/src/dsql/StmtNodes.cpp @@ -9010,6 +9010,9 @@ SetTransactionNode* SetTransactionNode::dsqlPass(DsqlCompilerScratch* dsqlScratc if (autoCommit.specified) dsqlScratch->appendUChar(isc_tpb_autocommit); + if (autoReleaseTempBlobID.specified) + dsqlScratch->appendUChar(isc_tpb_auto_release_temp_blobid); + if (lockTimeout.specified) { dsqlScratch->appendUChar(isc_tpb_lock_timeout); diff --git a/src/dsql/StmtNodes.h b/src/dsql/StmtNodes.h index b593528a11..60569094c3 100644 --- a/src/dsql/StmtNodes.h +++ b/src/dsql/StmtNodes.h @@ -1574,6 +1574,7 @@ public: NODE_PRINT(printer, ignoreLimbo); NODE_PRINT(printer, restartRequests); NODE_PRINT(printer, autoCommit); + NODE_PRINT(printer, autoReleaseTempBlobID); NODE_PRINT(printer, lockTimeout); //// FIXME-PRINT: NODE_PRINT(printer, reserveList); NODE_PRINT(printer, tpb); @@ -1600,6 +1601,7 @@ public: Nullable ignoreLimbo; Nullable restartRequests; Nullable autoCommit; + Nullable autoReleaseTempBlobID; }; diff --git a/src/dsql/parse.y b/src/dsql/parse.y index a53e5e4310..3afaa65f1a 100644 --- a/src/dsql/parse.y +++ b/src/dsql/parse.y @@ -343,10 +343,12 @@ using namespace Firebird; %token ACTION %token ADMIN +%token BLOBID %token CASCADE %token FREE_IT // ISC SQL extension %token RESTRICT %token ROLE +%token TEMP // New tokens added v6.0 @@ -5563,6 +5565,8 @@ tran_option($setTransactionNode) { setClause($setTransactionNode->restartRequests, "RESTART REQUESTS", true); } | AUTO COMMIT { setClause($setTransactionNode->autoCommit, "AUTO COMMIT", true); } + | AUTO RELEASE TEMP BLOBID + { setClause($setTransactionNode->autoReleaseTempBlobID, "AUTO RELEASE TEMP BLOBID", true); } // timeout | LOCK TIMEOUT nonneg_short_integer { setClause($setTransactionNode->lockTimeout, "LOCK TIMEOUT", (USHORT) $3); } @@ -9212,10 +9216,12 @@ non_reserved_word // added in FB 4.0.2 | BLOB_APPEND // added in FB 5.0 + | BLOBID | LOCKED | OPTIMIZE | QUARTER | TARGET + | TEMP | TIMEZONE_NAME | UNICODE_CHAR | UNICODE_VAL diff --git a/src/gpre/cmp.cpp b/src/gpre/cmp.cpp index e8bca853c7..2d2ddc7730 100644 --- a/src/gpre/cmp.cpp +++ b/src/gpre/cmp.cpp @@ -408,6 +408,8 @@ void CMP_t_start( gpre_tra* trans) *text++ = isc_tpb_autocommit; if (trans->tra_flags & TRA_no_auto_undo) *text++ = isc_tpb_no_auto_undo; + if (trans->tra_flags & TRA_auto_release_temp_blobid) + *text++ = isc_tpb_auto_release_temp_blobid; *text = 0; const USHORT tpb_len = text - tpb_buffer; diff --git a/src/gpre/gpre.h b/src/gpre/gpre.h index cd628bd33a..f90c3307f3 100644 --- a/src/gpre/gpre.h +++ b/src/gpre/gpre.h @@ -1414,7 +1414,8 @@ enum tra_flags_vals { TRA_read_committed = 32, TRA_autocommit = 64, TRA_rec_version = 128, - TRA_no_auto_undo = 256 + TRA_no_auto_undo = 256, + TRA_auto_release_temp_blobid = 512 }; const int MAX_TRA_OPTIONS = 8; diff --git a/src/gpre/hsh.h b/src/gpre/hsh.h index f7facbb5ac..ae86373066 100644 --- a/src/gpre/hsh.h +++ b/src/gpre/hsh.h @@ -43,6 +43,7 @@ {"AT", KW_AT}, {"AUTO", KW_AUTO}, {"AUTOCOMMIT", KW_AUTOCOMMIT}, + {"AUTO_RELEASE_TEMP_BLOBID", KW_AUTO_RELEASE_TEMP_BLOBID}, {"AVERAGE", KW_AVERAGE}, {"AVG", KW_AVERAGE}, {"\\", KW_BACK_SLASH}, diff --git a/src/gpre/sql.cpp b/src/gpre/sql.cpp index 5ae99b8830..5f73dc7eba 100644 --- a/src/gpre/sql.cpp +++ b/src/gpre/sql.cpp @@ -4424,6 +4424,12 @@ static act* act_set_transaction() continue; } + if (MSC_match(KW_AUTO_RELEASE_TEMP_BLOBID)) + { + trans->tra_flags |= TRA_auto_release_temp_blobid; + continue; + } + break; } diff --git a/src/gpre/words.h b/src/gpre/words.h index afd856f54c..5e89a60657 100644 --- a/src/gpre/words.h +++ b/src/gpre/words.h @@ -113,6 +113,7 @@ enum kwwords_t { KW_ASTERISK, KW_AUTO, KW_AUTOCOMMIT, + KW_AUTO_RELEASE_TEMP_BLOBID, KW_AVERAGE, KW_BASE_NAME, KW_BETWEEN, diff --git a/src/include/firebird/impl/consts_pub.h b/src/include/firebird/impl/consts_pub.h index d7f7004962..0989d4fe29 100644 --- a/src/include/firebird/impl/consts_pub.h +++ b/src/include/firebird/impl/consts_pub.h @@ -267,6 +267,7 @@ #define isc_tpb_lock_timeout 21 #define isc_tpb_read_consistency 22 #define isc_tpb_at_snapshot_number 23 +#define isc_tpb_auto_release_temp_blobid 24 /************************/ diff --git a/src/include/gen/Firebird.pas b/src/include/gen/Firebird.pas index be611bed58..85f6cbf579 100644 --- a/src/include/gen/Firebird.pas +++ b/src/include/gen/Firebird.pas @@ -4099,6 +4099,7 @@ const isc_tpb_lock_timeout = byte(21); isc_tpb_read_consistency = byte(22); isc_tpb_at_snapshot_number = byte(23); + isc_tpb_auto_release_temp_blobid = byte(24); isc_bpb_version1 = byte(1); isc_bpb_source_type = byte(1); isc_bpb_target_type = byte(2); diff --git a/src/jrd/blb.cpp b/src/jrd/blb.cpp index c0c247c8c8..071438d7a0 100644 --- a/src/jrd/blb.cpp +++ b/src/jrd/blb.cpp @@ -326,6 +326,9 @@ blb* blb::create2(thread_db* tdbb, blb* blob = allocate_blob(tdbb, transaction); + if (userBlob) + blob->blb_flags |= BLB_user; + if (type & isc_bpb_type_stream) blob->blb_flags |= BLB_stream; @@ -1296,7 +1299,9 @@ void blb::move(thread_db* tdbb, dsc* from_desc, dsc* to_desc, array->arr_request = own_request; } - blob->destroy(!materialized_blob); + const bool purgeBlob = !materialized_blob || + ((transaction->tra_flags & TRA_auto_release_temp_blobid) && (blob->blb_flags & BLB_user)); + blob->destroy(purgeBlob); } diff --git a/src/jrd/blb.h b/src/jrd/blb.h index 26176b5e69..cad5e0d836 100644 --- a/src/jrd/blb.h +++ b/src/jrd/blb.h @@ -181,6 +181,7 @@ const int BLB_seek = 32; // Seek is pending const int BLB_large_scan = 64; // Blob is larger than page buffer cache const int BLB_close_on_read = 128; // Temporary blob is not closed until read const int BLB_bulk = 256; // Blob created by bulk insert operation +const int BLB_user = 512; // User-defined blob /* Blob levels are: diff --git a/src/jrd/tra.cpp b/src/jrd/tra.cpp index 8e73b6ad64..baf4a5f92c 100644 --- a/src/jrd/tra.cpp +++ b/src/jrd/tra.cpp @@ -3162,6 +3162,10 @@ static void transaction_options(thread_db* tdbb, transaction->tra_flags |= TRA_no_auto_undo; break; + case isc_tpb_auto_release_temp_blobid: + transaction->tra_flags |= TRA_auto_release_temp_blobid; + break; + case isc_tpb_lock_write: // Cannot set a R/W table reservation if the whole txn is R/O. if (read_only.value) diff --git a/src/jrd/tra.h b/src/jrd/tra.h index dea0202e96..35a1797b27 100644 --- a/src/jrd/tra.h +++ b/src/jrd/tra.h @@ -435,10 +435,11 @@ const ULONG TRA_read_consistency = 0x40000L; // ensure read consistency in this const ULONG TRA_ex_restart = 0x80000L; // Exception was raised to restart request const ULONG TRA_replicating = 0x100000L; // transaction is allowed to be replicated const ULONG TRA_no_blob_check = 0x200000L; // disable blob access checking +const ULONG TRA_auto_release_temp_blobid = 0x400000L; // remove temp ids of materialized user blobs from tra_blobs // flags derived from TPB, see also transaction_options() at tra.cpp const ULONG TRA_OPTIONS_MASK = (TRA_degree3 | TRA_readonly | TRA_ignore_limbo | TRA_read_committed | - TRA_autocommit | TRA_rec_version | TRA_read_consistency | TRA_no_auto_undo | TRA_restart_requests); + TRA_autocommit | TRA_rec_version | TRA_read_consistency | TRA_no_auto_undo | TRA_restart_requests | TRA_auto_release_temp_blobid); const int TRA_MASK = 3; //const int TRA_BITS_PER_TRANS = 2;