diff --git a/builds/install/misc/firebird.conf b/builds/install/misc/firebird.conf index 54711f45cf..c35907338d 100644 --- a/builds/install/misc/firebird.conf +++ b/builds/install/misc/firebird.conf @@ -1019,6 +1019,19 @@ #GCPolicy = combined +# ---------------------------- +# Maximum statement cache size +# +# The maximum amount of RAM used to cache unused DSQL compiled statements. +# If set to 0 (zero), statement cache is disabled. +# +# Per-database configurable. +# +# Type: integer +# +#MaxStatementCacheSize = 2M + + # ---------------------------- # Security database # diff --git a/builds/win32/msvc15/engine.vcxproj b/builds/win32/msvc15/engine.vcxproj index 94e4e4222b..4bc3e762f6 100644 --- a/builds/win32/msvc15/engine.vcxproj +++ b/builds/win32/msvc15/engine.vcxproj @@ -42,6 +42,7 @@ + @@ -190,6 +191,7 @@ + diff --git a/builds/win32/msvc15/engine.vcxproj.filters b/builds/win32/msvc15/engine.vcxproj.filters index 3356f663a6..80fe264b9f 100644 --- a/builds/win32/msvc15/engine.vcxproj.filters +++ b/builds/win32/msvc15/engine.vcxproj.filters @@ -144,6 +144,9 @@ DSQL + + DSQL + DSQL @@ -560,6 +563,9 @@ Header files + + Header files + Header files diff --git a/src/common/classes/alloc.cpp b/src/common/classes/alloc.cpp index 86c5a64ec9..efe4b49b2a 100644 --- a/src/common/classes/alloc.cpp +++ b/src/common/classes/alloc.cpp @@ -1850,6 +1850,11 @@ public: // Create memory pool instance static MemPool* createPool(MemPool* parent, MemoryStats& stats); + MemoryStats& getStatsGroup() noexcept + { + return *stats; + } + // Set statistics group for pool. Usage counters will be decremented from // previously set group and added to new void setStatsGroup(MemoryStats& stats) noexcept; @@ -2262,6 +2267,11 @@ void MemPool::setStatsGroup(MemoryStats& newStats) noexcept stats->increment_usage(sav_used_memory); } +MemoryStats& MemoryPool::getStatsGroup() noexcept +{ + return pool->getStatsGroup(); +} + void MemoryPool::setStatsGroup(MemoryStats& newStats) noexcept { pool->setStatsGroup(newStats); diff --git a/src/common/classes/alloc.h b/src/common/classes/alloc.h index 470ea439b9..d133837206 100644 --- a/src/common/classes/alloc.h +++ b/src/common/classes/alloc.h @@ -213,6 +213,8 @@ public: // Get context pool for current thread of execution static MemoryPool* getContextPool(); + MemoryStats& getStatsGroup() noexcept; + // Set statistics group for pool. Usage counters will be decremented from // previously set group and added to new void setStatsGroup(MemoryStats& stats) noexcept; diff --git a/src/common/config/config.cpp b/src/common/config/config.cpp index 6e17cf9b50..1e43ea4769 100644 --- a/src/common/config/config.cpp +++ b/src/common/config/config.cpp @@ -410,6 +410,8 @@ void Config::checkValues() checkIntForHiBound(KEY_TIP_CACHE_BLOCK_SIZE, MAX_ULONG, true); checkIntForLoBound(KEY_INLINE_SORT_THRESHOLD, 0, true); + + checkIntForLoBound(KEY_MAX_STATEMENT_CACHE_SIZE, 0, true); } diff --git a/src/common/config/config.h b/src/common/config/config.h index c5b86233d1..a7b1febc4c 100644 --- a/src/common/config/config.h +++ b/src/common/config/config.h @@ -188,6 +188,7 @@ enum ConfigKey KEY_USE_FILESYSTEM_CACHE, KEY_INLINE_SORT_THRESHOLD, KEY_TEMP_PAGESPACE_DIR, + KEY_MAX_STATEMENT_CACHE_SIZE, MAX_CONFIG_KEY // keep it last }; @@ -302,7 +303,8 @@ constexpr ConfigEntry entries[MAX_CONFIG_KEY] = {TYPE_STRING, "DataTypeCompatibility", false, nullptr}, {TYPE_BOOLEAN, "UseFileSystemCache", false, true}, {TYPE_INTEGER, "InlineSortThreshold", false, 1000}, // bytes - {TYPE_STRING, "TempTableDirectory", false, ""} + {TYPE_STRING, "TempTableDirectory", false, ""}, + {TYPE_INTEGER, "MaxStatementCacheSize", false, 2 * 1048576} // bytes }; @@ -624,6 +626,8 @@ public: CONFIG_GET_PER_DB_KEY(ULONG, getInlineSortThreshold, KEY_INLINE_SORT_THRESHOLD, getInt); CONFIG_GET_PER_DB_STR(getTempPageSpaceDirectory, KEY_TEMP_PAGESPACE_DIR); + + CONFIG_GET_PER_DB_INT(getMaxStatementCacheSize, KEY_MAX_STATEMENT_CACHE_SIZE); }; // Implementation of interface to access master configuration file diff --git a/src/dsql/DsqlRequests.cpp b/src/dsql/DsqlRequests.cpp index c5b34e8503..2bdb14f154 100644 --- a/src/dsql/DsqlRequests.cpp +++ b/src/dsql/DsqlRequests.cpp @@ -23,6 +23,7 @@ #include "../dsql/DsqlRequests.h" #include "../dsql/dsql.h" #include "../dsql/DsqlBatch.h" +#include "../dsql/DsqlStatementCache.h" #include "../dsql/Nodes.h" #include "../jrd/Statement.h" #include "../jrd/req.h" @@ -1035,6 +1036,8 @@ void DsqlDdlRequest::execute(thread_db* tdbb, jrd_tra** traHandle, { AutoSetRestoreFlag execDdl(&tdbb->tdbb_flags, TDBB_repl_in_progress, true); + req_dbb->dbb_attachment->att_dsql_instance->dbb_statement_cache->purgeAllAttachments(tdbb); + node->executeDdl(tdbb, internalScratch, req_transaction); const bool isInternalRequest = diff --git a/src/dsql/DsqlRequests.h b/src/dsql/DsqlRequests.h index 451936a900..860f46433a 100644 --- a/src/dsql/DsqlRequests.h +++ b/src/dsql/DsqlRequests.h @@ -75,11 +75,6 @@ public: return nullptr; } - virtual bool isDml() const - { - return false; - } - virtual DsqlCursor* openCursor(thread_db* tdbb, jrd_tra** traHandle, Firebird::IMessageMetadata* inMeta, const UCHAR* inMsg, Firebird::IMessageMetadata* outMeta, ULONG flags) @@ -165,11 +160,6 @@ public: return request; } - bool isDml() const override - { - return true; - } - DsqlCursor* openCursor(thread_db* tdbb, jrd_tra** traHandle, Firebird::IMessageMetadata* inMeta, const UCHAR* inMsg, Firebird::IMessageMetadata* outMeta, ULONG flags) override; diff --git a/src/dsql/DsqlStatementCache.cpp b/src/dsql/DsqlStatementCache.cpp new file mode 100644 index 0000000000..ecec0e2c7a --- /dev/null +++ b/src/dsql/DsqlStatementCache.cpp @@ -0,0 +1,285 @@ +/* + * The contents of this file are subject to the Initial + * Developer's Public License Version 1.0 (the "License"); + * you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * http://www.ibphoenix.com/main.nfs?a=ibphoenix&page=ibp_idpl. + * + * Software distributed under the License is distributed AS IS, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. + * See the License for the specific language governing rights + * and limitations under the License. + * + * The Original Code was created by Adriano dos Santos Fernandes + * for the Firebird Open Source RDBMS project. + * + * Copyright (c) 2022 Adriano dos Santos Fernandes + * and all contributors signed below. + * + * All Rights Reserved. + * Contributor(s): ______________________________________. + */ + +#include "firebird.h" +#include "../dsql/DsqlStatementCache.h" +#include "../dsql/DsqlStatements.h" +#include "../jrd/Attachment.h" +#include "../jrd/Statement.h" +#include "../jrd/lck.h" +#include "../jrd/lck_proto.h" + +using namespace Firebird; +using namespace Jrd; + + +// Class DsqlStatementCache + +DsqlStatementCache::DsqlStatementCache(MemoryPool& o, Attachment* attachment) + : PermanentStorage(o), + map(o), + activeStatementList(o), + inactiveStatementList(o) +{ + const auto dbb = attachment->att_database; + maxCacheSize = dbb->dbb_config->getMaxStatementCacheSize(); +} + +DsqlStatementCache::~DsqlStatementCache() +{ + purge(JRD_get_thread_data()); +} + +int DsqlStatementCache::blockingAst(void* astObject) +{ +#ifdef DSQL_STATEMENT_CACHE_DEBUG + printf("DsqlStatementCache::blockingAst()\n"); +#endif + + const auto self = static_cast(astObject); + + try + { + const auto dbb = self->lock->lck_dbb; + AsyncContextHolder tdbb(dbb, FB_FUNCTION, self->lock); + + self->purge(tdbb); + } + catch (const Exception&) + {} // no-op + + return 0; +} + +RefPtr DsqlStatementCache::getStatement(thread_db* tdbb, const string& text, USHORT clientDialect, + bool isInternalRequest) +{ + RefStrPtr key; + buildStatementKey(tdbb, key, text, clientDialect, isInternalRequest); + + if (const auto entryPtr = map.get(key)) + { + const auto entry = *entryPtr; + auto dsqlStatement(entry->dsqlStatement); + + string verifyKey; + buildVerifyKey(tdbb, verifyKey, isInternalRequest); + + FB_SIZE_T verifyPos; + if (!entry->verifyCache.find(verifyKey, verifyPos)) + { + dsqlStatement->getStatement()->verifyAccess(tdbb); + entry->verifyCache.insert(verifyPos, verifyKey); + } + + if (!entry->active) + { + entry->dsqlStatement->setCacheKey(key); + // Active statement has cacheKey and will tell us when it's going to be released. + entry->dsqlStatement->release(); + + entry->active = true; + + cacheSize -= entry->size; + + activeStatementList.splice(activeStatementList.end(), inactiveStatementList, entry); + } + +#ifdef DSQL_STATEMENT_CACHE_DEBUG + dump(); +#endif + + return dsqlStatement; + } + + return {}; +} + +void DsqlStatementCache::putStatement(thread_db* tdbb, const string& text, USHORT clientDialect, + bool isInternalRequest, RefPtr dsqlStatement) +{ + fb_assert(dsqlStatement->isDml()); + + const unsigned statementSize = dsqlStatement->getSize(); + + RefStrPtr key; + buildStatementKey(tdbb, key, text, clientDialect, isInternalRequest); + + StatementEntry newStatement(getPool()); + newStatement.key = key; + newStatement.size = statementSize; + newStatement.dsqlStatement = std::move(dsqlStatement); + newStatement.active = true; + + string verifyKey; + buildVerifyKey(tdbb, verifyKey, isInternalRequest); + newStatement.verifyCache.add(verifyKey); + + newStatement.dsqlStatement->setCacheKey(key); + // Active statement has cacheKey and will tell us when it's going to be released. + newStatement.dsqlStatement->release(); + + activeStatementList.pushBack(std::move(newStatement)); + map.put(key, --activeStatementList.end()); + + if (!lock) + { + lock = FB_NEW_RPT(getPool(), 0) Lock(tdbb, 0, LCK_dsql_statement_cache, this, blockingAst); + LCK_lock(tdbb, lock, LCK_SR, LCK_WAIT); + } + +#ifdef DSQL_STATEMENT_CACHE_DEBUG + dump(); +#endif +} + +void DsqlStatementCache::statementGoingInactive(Firebird::RefStrPtr& key) +{ + const auto entryPtr = map.get(key); + + if (!entryPtr) + { + fb_assert(false); + return; + } + + const auto entry = *entryPtr; + + fb_assert(entry->active); + entry->active = false; + entry->size = entry->dsqlStatement->getSize(); // update size + + inactiveStatementList.splice(inactiveStatementList.end(), activeStatementList, entry); + + cacheSize += entry->size; + + if (cacheSize > maxCacheSize) + shrink(); +} + +void DsqlStatementCache::purge(thread_db* tdbb) +{ + for (auto& entry : activeStatementList) + { + entry.dsqlStatement->addRef(); + entry.dsqlStatement->resetCacheKey(); + } + + map.clear(); + activeStatementList.clear(); + inactiveStatementList.clear(); + + cacheSize = 0; + + if (lock) + { + LCK_release(tdbb, lock); + lock.reset(); + } +} + +void DsqlStatementCache::purgeAllAttachments(thread_db* tdbb) +{ + if (lock) + LCK_convert(tdbb, lock, LCK_EX, LCK_WAIT); + else + { + lock = FB_NEW_RPT(getPool(), 0) Lock(tdbb, 0, LCK_dsql_statement_cache, this, blockingAst); + LCK_lock(tdbb, lock, LCK_EX, LCK_WAIT); + } + + purge(tdbb); +} + +void DsqlStatementCache::buildStatementKey(thread_db* tdbb, RefStrPtr& key, const string& text, USHORT clientDialect, + bool isInternalRequest) +{ + const auto attachment = tdbb->getAttachment(); + + const SSHORT charSetId = isInternalRequest ? CS_METADATA : attachment->att_charset; + + key = FB_NEW_POOL(getPool()) RefString(getPool()); + + key->resize(1 + sizeof(charSetId) + text.length()); + char* p = key->begin(); + *p = (clientDialect << 1) | int(isInternalRequest); + memcpy(p + 1, &charSetId, sizeof(charSetId)); + memcpy(p + 1 + sizeof(charSetId), text.c_str(), text.length()); +} + +void DsqlStatementCache::buildVerifyKey(thread_db* tdbb, string& key, bool isInternalRequest) +{ + key.clear(); + + const auto attachment = tdbb->getAttachment(); + + if (isInternalRequest || !attachment->att_user) + return; + + const auto& roles = attachment->att_user->getGrantedRoles(tdbb); + + string roleStr; + + for (const auto& role : roles) + { + roleStr.printf("%d,%s,", int(role.length()), role.c_str()); + key += roleStr; + } +} + +void DsqlStatementCache::shrink() +{ +#ifdef DSQL_STATEMENT_CACHE_DEBUG + printf("DsqlStatementCache::shrink() - cacheSize: %u, maxCacheSize: %u\n\n", cacheSize, maxCacheSize); +#endif + + while (cacheSize > maxCacheSize && !inactiveStatementList.isEmpty()) + { + const auto& front = inactiveStatementList.front(); + map.remove(front.key); + cacheSize -= front.size; + inactiveStatementList.erase(inactiveStatementList.begin()); + } + +#ifdef DSQL_STATEMENT_CACHE_DEBUG + dump(); +#endif +} + +#ifdef DSQL_STATEMENT_CACHE_DEBUG +void DsqlStatementCache::dump() +{ + printf("DsqlStatementCache::dump() - cacheSize: %u, maxCacheSize: %u\n\n", cacheSize, maxCacheSize); + + printf("\tactive:\n"); + + for (auto& entry : activeStatementList) + printf("\t\tsize: %u; text: %s\n", entry.size, entry.dsqlStatement->getSqlText()->c_str()); + + printf("\n\tinactive:\n"); + + for (auto& entry : inactiveStatementList) + printf("\t\tsize: %u; text: %s\n", entry.size, entry.dsqlStatement->getSqlText()->c_str()); + + printf("\n"); +} +#endif diff --git a/src/dsql/DsqlStatementCache.h b/src/dsql/DsqlStatementCache.h new file mode 100644 index 0000000000..da6e3f793c --- /dev/null +++ b/src/dsql/DsqlStatementCache.h @@ -0,0 +1,137 @@ +/* + * The contents of this file are subject to the Initial + * Developer's Public License Version 1.0 (the "License"); + * you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * http://www.ibphoenix.com/main.nfs?a=ibphoenix&page=ibp_idpl. + * + * Software distributed under the License is distributed AS IS, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. + * See the License for the specific language governing rights + * and limitations under the License. + * + * The Original Code was created by Adriano dos Santos Fernandes + * for the Firebird Open Source RDBMS project. + * + * Copyright (c) 2022 Adriano dos Santos Fernandes + * and all contributors signed below. + * + * All Rights Reserved. + * Contributor(s): ______________________________________. + */ + +#ifndef DSQL_STATEMENT_CACHE_H +#define DSQL_STATEMENT_CACHE_H + +///#define DSQL_STATEMENT_CACHE_DEBUG 1 + +#include "../common/classes/alloc.h" +#include "../common/classes/DoublyLinkedList.h" +#include "../common/classes/fb_string.h" +#include "../common/classes/GenericMap.h" +#include "../common/classes/objects_array.h" +#include "../common/classes/RefCounted.h" + +namespace Jrd { + + +class Attachment; +class DsqlStatement; +class Lock; +class thread_db; + + +class DsqlStatementCache final : public Firebird::PermanentStorage +{ +private: + struct StatementEntry + { + explicit StatementEntry(MemoryPool& p) + : verifyCache(p) + { + } + + StatementEntry(MemoryPool& p, StatementEntry&& o) + : key(std::move(o.key)), + dsqlStatement(std::move(o.dsqlStatement)), + verifyCache(p, std::move(o.verifyCache)), + size(o.size), + active(o.active) + { + } + + StatementEntry(const StatementEntry&) = delete; + StatementEntry& operator=(const StatementEntry&) = delete; + + Firebird::RefStrPtr key; + Firebird::RefPtr dsqlStatement; + Firebird::SortedObjectsArray verifyCache; + unsigned size = 0; + bool active = true; + }; + + class RefStrPtrComparator + { + public: + static bool greaterThan(const Firebird::RefStrPtr& i1, const Firebird::RefStrPtr& i2) + { + return *i1 > *i2; + } + }; + +public: + explicit DsqlStatementCache(MemoryPool& o, Attachment* attachment); + ~DsqlStatementCache(); + + DsqlStatementCache(const DsqlStatementCache&) = delete; + DsqlStatementCache& operator=(const DsqlStatementCache&) = delete; + +private: + static int blockingAst(void* astObject); + +public: + bool isActive() const + { + return maxCacheSize > 0; + } + + Firebird::RefPtr getStatement(thread_db* tdbb, const Firebird::string& text, + USHORT clientDialect, bool isInternalRequest); + + void putStatement(thread_db* tdbb, const Firebird::string& text, USHORT clientDialect, bool isInternalRequest, + Firebird::RefPtr dsqlStatement); + + void statementGoingInactive(Firebird::RefStrPtr& key); + + void purge(thread_db* tdbb); + void purgeAllAttachments(thread_db* tdbb); + +private: + void buildStatementKey(thread_db* tdbb, Firebird::RefStrPtr& key, const Firebird::string& text, + USHORT clientDialect, bool isInternalRequest); + + void buildVerifyKey(thread_db* tdbb, Firebird::string& key, bool isInternalRequest); + + void shrink(); + +#ifdef DSQL_STATEMENT_CACHE_DEBUG + void dump(); +#endif + +private: + Firebird::NonPooledMap< + Firebird::RefStrPtr, + Firebird::DoublyLinkedList::Iterator, + RefStrPtrComparator + > map; + Firebird::DoublyLinkedList activeStatementList; + Firebird::DoublyLinkedList inactiveStatementList; + Firebird::AutoPtr lock; + unsigned maxCacheSize = 0; + unsigned cacheSize = 0; +}; + + +} // namespace Jrd + +#endif // DSQL_STATEMENT_CACHE_H diff --git a/src/dsql/DsqlStatements.cpp b/src/dsql/DsqlStatements.cpp index 9f51ce4153..9d18ac74b7 100644 --- a/src/dsql/DsqlStatements.cpp +++ b/src/dsql/DsqlStatements.cpp @@ -24,6 +24,7 @@ #include "../dsql/dsql.h" #include "../dsql/Nodes.h" #include "../dsql/DsqlCompilerScratch.h" +#include "../dsql/DsqlStatementCache.h" #include "../jrd/Statement.h" #include "../dsql/errd_proto.h" #include "../dsql/gen_proto.h" @@ -58,12 +59,22 @@ void DsqlStatement::rethrowDdlException(status_exception& ex, bool metadataUpdat int DsqlStatement::release() { fb_assert(refCounter.value() > 0); - const int refCnt = --refCounter; + int refCnt = --refCounter; if (!refCnt) { - doRelease(); - dsqlAttachment->deletePool(&getPool()); + if (cacheKey) + { + refCnt = ++refCounter; + auto key = cacheKey; + cacheKey = nullptr; + dsqlAttachment->dbb_statement_cache->statementGoingInactive(key); + } + else + { + doRelease(); + dsqlAttachment->deletePool(&getPool()); + } } return refCnt; @@ -119,6 +130,11 @@ void DsqlDmlStatement::doRelease() DsqlStatement::doRelease(); } +unsigned DsqlDmlStatement::getSize() const +{ + return DsqlStatement::getSize() + statement->getSize(); +} + void DsqlDmlStatement::dsqlPass(thread_db* tdbb, DsqlCompilerScratch* scratch, ntrace_result_t* traceResult) { { // scope diff --git a/src/dsql/DsqlStatements.h b/src/dsql/DsqlStatements.h index fc88b53662..42d18d01a1 100644 --- a/src/dsql/DsqlStatements.h +++ b/src/dsql/DsqlStatements.h @@ -26,6 +26,7 @@ #include "../common/classes/array.h" #include "../common/classes/fb_string.h" #include "../common/classes/NestConst.h" +#include "../common/classes/RefCounted.h" #include "../jrd/jrd.h" #include "../jrd/ntrace.h" #include "../dsql/DsqlRequests.h" @@ -67,14 +68,15 @@ public: static void rethrowDdlException(Firebird::status_exception& ex, bool metadataUpdate, DdlNode* node); public: - DsqlStatement(MemoryPool& p, dsql_dbb* aDsqlAttachment) - : PermanentStorage(p), + DsqlStatement(MemoryPool& pool, dsql_dbb* aDsqlAttachment) + : PermanentStorage(pool), dsqlAttachment(aDsqlAttachment), type(TYPE_SELECT), flags(0), blrVersion(5), - ports(p) + ports(pool) { + pool.setStatsGroup(memoryStats); } protected: @@ -133,7 +135,15 @@ public: const dsql_par* getEof() const { return eof; } void setEof(dsql_par* value) { eof = value; } + void setCacheKey(Firebird::RefStrPtr& value) { cacheKey = value; } + void resetCacheKey() { cacheKey = nullptr; } + public: + virtual bool isDml() const + { + return false; + } + virtual Statement* getStatement() const { return nullptr; @@ -149,6 +159,11 @@ public: return true; } + virtual unsigned getSize() const + { + return (unsigned) memoryStats.getCurrentUsage(); + } + virtual void dsqlPass(thread_db* tdbb, DsqlCompilerScratch* scratch, ntrace_result_t* traceResult) = 0; virtual DsqlRequest* createRequest(thread_db* tdbb, dsql_dbb* dbb) = 0; @@ -157,11 +172,13 @@ protected: protected: dsql_dbb* dsqlAttachment; + Firebird::MemoryStats memoryStats; Type type; // Type of statement ULONG flags; // generic flag unsigned blrVersion; Firebird::RefStrPtr sqlText; Firebird::RefStrPtr orgText; + Firebird::RefStrPtr cacheKey; Firebird::Array ports; // Port messages dsql_msg* sendMsg = nullptr; // Message to be sent to start request dsql_msg* receiveMsg = nullptr; // Per record message to be received @@ -182,11 +199,18 @@ public: } public: + bool isDml() const override + { + return true; + } + Statement* getStatement() const override { return statement; } + unsigned getSize() const override; + void dsqlPass(thread_db* tdbb, DsqlCompilerScratch* scratch, ntrace_result_t* traceResult) override; DsqlDmlRequest* createRequest(thread_db* tdbb, dsql_dbb* dbb) override; diff --git a/src/dsql/dsql.cpp b/src/dsql/dsql.cpp index 1f77edb1b8..0d60b0aca8 100644 --- a/src/dsql/dsql.cpp +++ b/src/dsql/dsql.cpp @@ -72,6 +72,7 @@ #include "../common/utils_proto.h" #include "../common/StatusArg.h" #include "../dsql/DsqlBatch.h" +#include "../dsql/DsqlStatementCache.h" #ifdef HAVE_CTYPE_H #include @@ -109,6 +110,21 @@ namespace IMPLEMENT_TRACE_ROUTINE(dsql_trace, "DSQL") #endif +dsql_dbb::dsql_dbb(MemoryPool& p, Attachment* attachment) + : dbb_relations(p), + dbb_procedures(p), + dbb_functions(p), + dbb_charsets(p), + dbb_collations(p), + dbb_charsets_by_id(p), + dbb_cursors(p), + dbb_pool(p), + dbb_dfl_charset(p) +{ + dbb_attachment = attachment; + dbb_statement_cache = FB_NEW_POOL(p) DsqlStatementCache(p, dbb_attachment); +} + dsql_dbb::~dsql_dbb() { } @@ -411,8 +427,7 @@ static dsql_dbb* init(thread_db* tdbb, Jrd::Attachment* attachment) return attachment->att_dsql_instance; MemoryPool& pool = *attachment->createPool(); - dsql_dbb* const database = FB_NEW_POOL(pool) dsql_dbb(pool); - database->dbb_attachment = attachment; + dsql_dbb* const database = FB_NEW_POOL(pool) dsql_dbb(pool, attachment); attachment->att_dsql_instance = database; INI_init_dsql(tdbb, database); @@ -497,12 +512,24 @@ static RefPtr prepareStatement(thread_db* tdbb, dsql_dbb* databas Arg::Gds(isc_sql_too_long) << Arg::Num(MAX_SQL_LENGTH)); } + string textStr(text, textLength); + const bool isStatementCacheActive = database->dbb_statement_cache->isActive(); + + RefPtr dsqlStatement; + + if (isStatementCacheActive) + { + dsqlStatement = database->dbb_statement_cache->getStatement(tdbb, textStr, clientDialect, isInternalRequest); + + if (dsqlStatement) + return dsqlStatement; + } + // allocate the statement block, then prepare the statement MemoryPool* scratchPool = nullptr; DsqlCompilerScratch* scratch = nullptr; MemoryPool* statementPool = database->createPool(); - RefPtr dsqlStatement; Jrd::ContextPoolHolder statementContext(tdbb, statementPool); try @@ -593,6 +620,12 @@ static RefPtr prepareStatement(thread_db* tdbb, dsql_dbb* databas if (!isInternalRequest && dsqlStatement->mustBeReplicated()) dsqlStatement->setOrgText(text, textLength); + if (isStatementCacheActive && dsqlStatement->isDml()) + { + database->dbb_statement_cache->putStatement(tdbb, + textStr, clientDialect, isInternalRequest, dsqlStatement); + } + return dsqlStatement; } catch (const Exception&) diff --git a/src/dsql/dsql.h b/src/dsql/dsql.h index 7ca31a3bc1..067ae93c61 100644 --- a/src/dsql/dsql.h +++ b/src/dsql/dsql.h @@ -77,12 +77,9 @@ namespace Jrd class Attachment; class Database; class DsqlCompilerScratch; - class DsqlDmlStatement; - class DdlNode; + class DsqlStatement; + class DsqlStatementCache; class RseNode; - class StmtNode; - class TransactionNode; - class SessionManagementNode; class ValueExprNode; class ValueListNode; class WindowClause; @@ -92,7 +89,6 @@ namespace Jrd struct bid; class dsql_ctx; - class dsql_msg; class dsql_par; class dsql_map; class dsql_intlsym; @@ -130,24 +126,14 @@ public: Firebird::LeftPooledMap dbb_collations; // known collations in database Firebird::NonPooledMap dbb_charsets_by_id; // charsets sorted by charset_id Firebird::LeftPooledMap dbb_cursors; // known cursors in database + Firebird::AutoPtr dbb_statement_cache; MemoryPool& dbb_pool; // The current pool for the dbb Attachment* dbb_attachment; MetaName dbb_dfl_charset; bool dbb_no_charset; - explicit dsql_dbb(MemoryPool& p) - : dbb_relations(p), - dbb_procedures(p), - dbb_functions(p), - dbb_charsets(p), - dbb_collations(p), - dbb_charsets_by_id(p), - dbb_cursors(p), - dbb_pool(p), - dbb_dfl_charset(p) - {} - + dsql_dbb(MemoryPool& p, Attachment* attachment); ~dsql_dbb(); MemoryPool* createPool() diff --git a/src/jrd/Attachment.cpp b/src/jrd/Attachment.cpp index 9d3cbd5915..fcb946a9b6 100644 --- a/src/jrd/Attachment.cpp +++ b/src/jrd/Attachment.cpp @@ -145,6 +145,8 @@ void Jrd::Attachment::destroy(Attachment* const attachment) MemoryPool* Jrd::Attachment::createPool() { MemoryPool* const pool = MemoryPool::createPool(att_pool, att_memory_stats); + auto stats = FB_NEW_POOL(*pool) MemoryStats(&att_memory_stats); + pool->setStatsGroup(*stats); att_pools.add(pool); return pool; } @@ -154,9 +156,7 @@ void Jrd::Attachment::deletePool(MemoryPool* pool) { if (pool) { - FB_SIZE_T pos; - if (att_pools.find(pool, pos)) - att_pools.remove(pos); + att_pools.findAndRemove(pool); #ifdef DEBUG_LCK_LIST // hvlad: this could be slow, use only when absolutely necessary diff --git a/src/jrd/Monitoring.cpp b/src/jrd/Monitoring.cpp index 17aa3438ec..ece65b9d30 100644 --- a/src/jrd/Monitoring.cpp +++ b/src/jrd/Monitoring.cpp @@ -1172,7 +1172,13 @@ void Monitoring::putStatement(SnapshotData::DumpRecord& record, const Statement* record.storeInteger(f_mon_cmp_stmt_type, obj_trigger); } + // statistics + const int stat_id = fb_utils::genUniqueId(); + record.storeGlobalId(f_mon_cmp_stmt_stat_id, getGlobalId(stat_id)); + record.write(); + + putMemoryUsage(record, statement->pool->getStatsGroup(), stat_id, stat_cmp_statement); } diff --git a/src/jrd/Statement.cpp b/src/jrd/Statement.cpp index 0ac1683f8d..b876daced9 100644 --- a/src/jrd/Statement.cpp +++ b/src/jrd/Statement.cpp @@ -74,8 +74,7 @@ Statement::Statement(thread_db* tdbb, MemoryPool* p, CompilerScratch* csb) localTables(*p), invariants(*p), blr(*p), - mapFieldInfo(*p), - mapItemInfo(*p) + mapFieldInfo(*p) { try { @@ -86,9 +85,16 @@ Statement::Statement(thread_db* tdbb, MemoryPool* p, CompilerScratch* csb) static_cast(csb->csb_node) : NULL; accessList = csb->csb_access; + csb->csb_access.clear(); + externalList = csb->csb_external; + csb->csb_external.clear(); + mapFieldInfo.takeOwnership(csb->csb_map_field_info); + resources = csb->csb_resources; // Assign array contents + csb->csb_resources.clear(); + impureSize = csb->csb_impure; //if (csb->csb_g_flags & csb_blr_version4) @@ -155,19 +161,22 @@ Statement::Statement(thread_db* tdbb, MemoryPool* p, CompilerScratch* csb) // make a vector of all used RSEs fors = csb->csb_fors; + csb->csb_fors.clear(); localTables = csb->csb_localTables; + csb->csb_localTables.clear(); // make a vector of all invariant-type nodes, so that we will // be able to easily reinitialize them when we restart the request invariants.join(csb->csb_invariants); + csb->csb_invariants.clear(); rpbsSetup.grow(csb->csb_n_stream); - CompilerScratch::csb_repeat* tail = csb->csb_rpt.begin(); - const CompilerScratch::csb_repeat* const streams_end = tail + csb->csb_n_stream; + auto tail = csb->csb_rpt.begin(); + const auto* const streams_end = tail + csb->csb_n_stream; - for (record_param* rpb = rpbsSetup.begin(); tail < streams_end; ++rpb, ++tail) + for (auto rpb = rpbsSetup.begin(); tail < streams_end; ++rpb, ++tail) { // fetch input stream for update if all booleans matched against indices if ((tail->csb_flags & csb_update) && !(tail->csb_flags & csb_unmatched)) @@ -186,6 +195,22 @@ Statement::Statement(thread_db* tdbb, MemoryPool* p, CompilerScratch* csb) delete tail->csb_fields; tail->csb_fields = NULL; } + + if (csb->csb_variables) + csb->csb_variables->clear(); + + csb->csb_current_nodes.free(); + csb->csb_current_for_nodes.free(); + csb->csb_computing_fields.free(); + csb->csb_variables_used_in_subroutines.free(); + csb->csb_dbg_info.reset(); + csb->csb_map_item_info.clear(); + csb->csb_message_pad.clear(); + csb->subFunctions.clear(); + csb->subProcedures.clear(); + csb->outerMessagesMap.clear(); + csb->outerVarsMap.clear(); + csb->csb_rpt.free(); } catch (Exception&) { @@ -206,13 +231,15 @@ Statement* Statement::makeStatement(thread_db* tdbb, CompilerScratch* csb, bool DEV_BLKCHK(csb, type_csb); SET_TDBB(tdbb); - Database* const dbb = tdbb->getDatabase(); + const auto dbb = tdbb->getDatabase(); fb_assert(dbb); - Request* const old_request = tdbb->getRequest(); - tdbb->setRequest(NULL); + const auto attachment = tdbb->getAttachment(); - Statement* statement = NULL; + const auto old_request = tdbb->getRequest(); + tdbb->setRequest(nullptr); + + Statement* statement = nullptr; try { @@ -277,7 +304,7 @@ Statement* Statement::makeStatement(thread_db* tdbb, CompilerScratch* csb, bool // Build the statement and the final request block. - MemoryPool* const pool = tdbb->getDefaultPool(); + const auto pool = tdbb->getDefaultPool(); statement = FB_NEW_POOL(*pool) Statement(tdbb, pool, csb); @@ -288,12 +315,8 @@ Statement* Statement::makeStatement(thread_db* tdbb, CompilerScratch* csb, bool if (statement) { // Release sub statements. - for (Statement** subStatement = statement->subStatements.begin(); - subStatement != statement->subStatements.end(); - ++subStatement) - { - (*subStatement)->release(tdbb); - } + for (auto subStatement : statement->subStatements) + subStatement->release(tdbb); } ex.stuffException(tdbb->tdbb_status_vector); @@ -302,9 +325,14 @@ Statement* Statement::makeStatement(thread_db* tdbb, CompilerScratch* csb, bool } if (internalFlag) + { statement->flags |= FLAG_INTERNAL; + statement->charSetId = CS_METADATA; + } + else + statement->charSetId = attachment->att_charset; - tdbb->getAttachment()->att_statements.add(statement); + attachment->att_statements.add(statement); return statement; } @@ -402,17 +430,11 @@ Request* Statement::getRequest(thread_db* tdbb, USHORT level) if (level < requests.getCount() && requests[level]) return requests[level]; - requests.grow(level + 1); - - MemoryStats* const parentStats = (flags & FLAG_INTERNAL) ? - &dbb->dbb_memory_stats : &attachment->att_memory_stats; - // Create the request. - Request* const request = FB_NEW_POOL(*pool) Request(attachment, this, parentStats); - - if (level == 0) - pool->setStatsGroup(request->req_memory_stats); + AutoMemoryPool reqPool(MemoryPool::createPool(pool)); + const auto request = FB_NEW_POOL(*reqPool) Request(reqPool, attachment, this); + requests.grow(level + 1); requests[level] = request; return request; @@ -500,15 +522,13 @@ void Statement::verifyAccess(thread_db* tdbb) if (!routine->getStatement()) continue; - for (const AccessItem* access = routine->getStatement()->accessList.begin(); - access != routine->getStatement()->accessList.end(); - ++access) + for (const auto& access : routine->getStatement()->accessList) { MetaName userName = item->user; - if (access->acc_ss_rel_id) + if (access.acc_ss_rel_id) { - const jrd_rel* view = MET_lookup_relation_id(tdbb, access->acc_ss_rel_id, false); + const jrd_rel* view = MET_lookup_relation_id(tdbb, access.acc_ss_rel_id, false); if (view && (view->rel_flags & REL_sql_relation)) userName = view->rel_owner_name; } @@ -517,17 +537,17 @@ void Statement::verifyAccess(thread_db* tdbb) UserId* effectiveUser = userName.hasData() ? attachment->getUserId(userName) : attachment->att_ss_user; AutoSetRestore userIdHolder(&attachment->att_ss_user, effectiveUser); - const SecurityClass* sec_class = SCL_get_class(tdbb, access->acc_security_name.c_str()); + const SecurityClass* sec_class = SCL_get_class(tdbb, access.acc_security_name.c_str()); if (routine->getName().package.isEmpty()) { SCL_check_access(tdbb, sec_class, aclType, routine->getName().identifier, - access->acc_mask, access->acc_type, true, access->acc_name, access->acc_r_name); + access.acc_mask, access.acc_type, true, access.acc_name, access.acc_r_name); } else { SCL_check_access(tdbb, sec_class, id_package, routine->getName().package, - access->acc_mask, access->acc_type, true, access->acc_name, access->acc_r_name); + access.acc_mask, access.acc_type, true, access.acc_name, access.acc_r_name); } } } @@ -650,7 +670,11 @@ void Statement::release(thread_db* tdbb) } for (Request** instance = requests.begin(); instance != requests.end(); ++instance) + { EXE_release(tdbb, *instance); + MemoryPool::deletePool((*instance)->req_pool); + *instance = nullptr; + } const auto attachment = tdbb->getAttachment(); @@ -842,41 +866,33 @@ template static void makeSubRoutines(thread_db* tdbb, Statement* st { typename T::Accessor subAccessor(&subs); - for (bool found = subAccessor.getFirst(); found; found = subAccessor.getNext()) + for (auto& sub : subs) { - typename T::ValueType subNode = subAccessor.current()->second; - Routine* subRoutine = subNode->routine; - CompilerScratch*& subCsb = subNode->subCsb; + auto subNode = sub.second; + auto subRoutine = subNode->routine; + auto& subCsb = subNode->subCsb; - Statement* subStatement = Statement::makeStatement(tdbb, subCsb, false); + auto subStatement = Statement::makeStatement(tdbb, subCsb, false); subStatement->parentStatement = statement; subRoutine->setStatement(subStatement); // Move dependencies and permissions from the sub routine to the parent. - for (CompilerScratch::Dependency* dependency = subCsb->csb_dependencies.begin(); - dependency != subCsb->csb_dependencies.end(); - ++dependency) - { - csb->csb_dependencies.push(*dependency); - } + for (auto& dependency : subCsb->csb_dependencies) + csb->csb_dependencies.push(dependency); - for (ExternalAccess* access = subCsb->csb_external.begin(); - access != subCsb->csb_external.end(); - ++access) + for (auto& access : subStatement->externalList) { FB_SIZE_T i; - if (!csb->csb_external.find(*access, i)) - csb->csb_external.insert(i, *access); + if (!csb->csb_external.find(access, i)) + csb->csb_external.insert(i, access); } - for (AccessItem* access = subCsb->csb_access.begin(); - access != subCsb->csb_access.end(); - ++access) + for (auto& access : subStatement->accessList) { FB_SIZE_T i; - if (!csb->csb_access.find(*access, i)) - csb->csb_access.insert(i, *access); + if (!csb->csb_access.find(access, i)) + csb->csb_access.insert(i, access); } delete subCsb; diff --git a/src/jrd/Statement.h b/src/jrd/Statement.h index cdf98ff9ad..6f68a360e1 100644 --- a/src/jrd/Statement.h +++ b/src/jrd/Statement.h @@ -55,6 +55,11 @@ public: return id; } + unsigned getSize() const + { + return (unsigned) pool->getStatsGroup().getCurrentUsage(); + } + const Routine* getRoutine() const; bool isActive() const; @@ -78,6 +83,7 @@ public: unsigned blrVersion; ULONG impureSize; // Size of impure area mutable StmtNumber id; // statement identifier + USHORT charSetId; // client character set (CS_METADATA for internal statements) Firebird::Array rpbsSetup; Firebird::Array requests; // vector of requests ExternalAccessList externalList; // Access to procedures/triggers to be checked @@ -96,7 +102,6 @@ public: Firebird::RefStrPtr sqlText; // SQL text (encoded in the metadata charset) Firebird::Array blr; // BLR for non-SQL query MapFieldInfo mapFieldInfo; // Map field name to field info - MapItemInfo mapItemInfo; // Map item to item info }; diff --git a/src/jrd/constants.h b/src/jrd/constants.h index 508a4bc33d..da2aa2f439 100644 --- a/src/jrd/constants.h +++ b/src/jrd/constants.h @@ -284,7 +284,8 @@ enum stat_group_t { stat_attachment = 1, stat_transaction = 2, stat_statement = 3, - stat_call = 4 + stat_call = 4, + stat_cmp_statement = 5 }; enum InfoType diff --git a/src/jrd/jrd.cpp b/src/jrd/jrd.cpp index af4b5a2302..1a92558460 100644 --- a/src/jrd/jrd.cpp +++ b/src/jrd/jrd.cpp @@ -134,6 +134,7 @@ #include "../dsql/dsql.h" #include "../dsql/dsql_proto.h" #include "../dsql/DsqlBatch.h" +#include "../dsql/DsqlStatementCache.h" #ifdef WIN_NT #include @@ -4714,13 +4715,9 @@ void JAttachment::transactRequest(CheckStatusWrapper* user_status, ITransaction* CompilerScratch* csb = PAR_parse(tdbb, reinterpret_cast(blr), blr_length, false); - request = Statement::makeRequest(tdbb, csb, false); - request->getStatement()->verifyAccess(tdbb); - for (FB_SIZE_T i = 0; i < csb->csb_rpt.getCount(); i++) { - const MessageNode* node = csb->csb_rpt[i].csb_message; - if (node) + if (const auto node = csb->csb_rpt[i].csb_message) { if (node->messageNumber == 0) inMessage = node; @@ -4728,6 +4725,9 @@ void JAttachment::transactRequest(CheckStatusWrapper* user_status, ITransaction* outMessage = node; } } + + request = Statement::makeRequest(tdbb, csb, false); + request->getStatement()->verifyAccess(tdbb); } catch (const Exception&) { @@ -7551,6 +7551,9 @@ void release_attachment(thread_db* tdbb, Jrd::Attachment* attachment) attachment->att_replicator = nullptr; + if (attachment->att_dsql_instance) + attachment->att_dsql_instance->dbb_statement_cache->purge(tdbb); + while (attachment->att_repl_appliers.hasData()) { AutoPtr cleanupApplier(attachment->att_repl_appliers.pop()); @@ -8906,8 +8909,10 @@ void thread_db::setRequest(Request* val) SSHORT thread_db::getCharSet() const { - if (request && request->charSetId != CS_dynamic) - return request->charSetId; + USHORT charSetId; + + if (request && (charSetId = request->getStatement()->charSetId) != CS_dynamic) + return charSetId; return attachment->att_charset; } diff --git a/src/jrd/lck.cpp b/src/jrd/lck.cpp index 62863b5e72..af2b76ab11 100644 --- a/src/jrd/lck.cpp +++ b/src/jrd/lck.cpp @@ -584,6 +584,7 @@ static lck_owner_t get_owner_type(enum lck_t lock_type) case LCK_record_gc: case LCK_alter_database: case LCK_repl_tables: + case LCK_dsql_statement_cache: owner_type = LCK_OWNER_attachment; break; diff --git a/src/jrd/lck.h b/src/jrd/lck.h index 892aa1fe21..1efa69cf26 100644 --- a/src/jrd/lck.h +++ b/src/jrd/lck.h @@ -74,7 +74,8 @@ enum lck_t { LCK_record_gc, // Record-level GC lock LCK_alter_database, // ALTER DATABASE lock LCK_repl_state, // Replication state lock - LCK_repl_tables // Replication set lock + LCK_repl_tables, // Replication set lock + LCK_dsql_statement_cache // DSQL statement cache lock }; // Lock owner types diff --git a/src/jrd/relations.h b/src/jrd/relations.h index 2fbfe56596..1016ad7c8e 100644 --- a/src/jrd/relations.h +++ b/src/jrd/relations.h @@ -752,4 +752,5 @@ RELATION(nam_mon_compiled_statements, rel_mon_compiled_statements, ODS_13_1, rel FIELD(f_mon_cmp_stmt_name, nam_mon_obj_name, fld_gnr_name, 0, ODS_13_1) FIELD(f_mon_cmp_stmt_type, nam_mon_obj_type, fld_obj_type, 0, ODS_13_1) FIELD(f_mon_cmp_stmt_pkg_name, nam_mon_pkg_name, fld_pkg_name, 0, ODS_13_1) + FIELD(f_mon_cmp_stmt_stat_id, nam_mon_stat_id, fld_stat_id, 0, ODS_13_1) END_RELATION diff --git a/src/jrd/req.h b/src/jrd/req.h index be5a82a55c..c6c266e562 100644 --- a/src/jrd/req.h +++ b/src/jrd/req.h @@ -266,11 +266,10 @@ private: }; public: - Request(Attachment* attachment, /*const*/ Statement* aStatement, - Firebird::MemoryStats* parent_stats) + Request(Firebird::AutoMemoryPool& pool, Attachment* attachment, /*const*/ Statement* aStatement) : statement(aStatement), - req_pool(statement->pool), - req_memory_stats(parent_stats), + req_pool(pool), + req_memory_stats(&aStatement->pool->getStatsGroup()), req_blobs(req_pool), req_stats(*req_pool), req_base_stats(*req_pool), @@ -288,6 +287,9 @@ public: setAttachment(attachment); req_rpb = statement->rpbsSetup; impureArea.grow(statement->impureSize); + + pool->setStatsGroup(req_memory_stats); + pool.release(); } Statement* getStatement() @@ -313,8 +315,6 @@ public: void setAttachment(Attachment* newAttachment) { req_attachment = newAttachment; - charSetId = statement->flags & Statement::FLAG_INTERNAL ? - CS_METADATA : req_attachment->att_charset; } bool isRoot() const @@ -351,9 +351,9 @@ private: public: MemoryPool* req_pool; + Firebird::MemoryStats req_memory_stats; Attachment* req_attachment; // database attachment USHORT req_incarnation; // incarnation number - Firebird::MemoryStats req_memory_stats; // Transaction pointer and doubly linked list pointers for requests in this // transaction. Maintained by TRA_attach_request/TRA_detach_request. @@ -399,7 +399,6 @@ public: SortOwner req_sorts; Firebird::Array req_rpb; // record parameter blocks Firebird::Array impureArea; // impure area - USHORT charSetId; // "client" character set of the request TriggerAction req_trigger_action; // action that caused trigger to fire // Fields to support read consistency in READ COMMITTED transactions diff --git a/src/jrd/scl.h b/src/jrd/scl.h index 14fc38eefb..caaef2f4eb 100644 --- a/src/jrd/scl.h +++ b/src/jrd/scl.h @@ -366,6 +366,13 @@ public: return usr_granted_roles.exist(role); } + const auto& getGrantedRoles(thread_db* tdbb) const + { + if (testFlag(USR_newrole)) + findGrantedRoles(tdbb); + return usr_granted_roles; + } + void makeRoleName(const int dialect) { makeRoleName(usr_sql_role_name, dialect); diff --git a/src/jrd/trace/TraceObjects.cpp b/src/jrd/trace/TraceObjects.cpp index 741882585a..93863a9acb 100644 --- a/src/jrd/trace/TraceObjects.cpp +++ b/src/jrd/trace/TraceObjects.cpp @@ -234,7 +234,7 @@ void TraceSQLStatementImpl::DSQLParamsImpl::fillParams() if (m_descs.getCount() || !m_params || m_params->getCount() == 0) return; - if (!m_stmt->isDml()) + if (!m_stmt->getDsqlStatement()->isDml()) { fb_assert(false); return;