firebird/firebird-mirror
firebird/firebird-mirror
8
0
Fork 1
mirror of https://github.com/FirebirdSQL/firebird.git synced 2025-01-22 18:43:02 +01:00
Code Issues

Compiled statement cache.

Browse Source
This commit is contained in:
Adriano dos Santos Fernandes 2022-03-04 20:44:33 -03:00 committed by Adriano dos Santos Fernandes
parent 0ecf1d71d5
commit 4c9fffd9b9
27 changed files with 670 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
builds/install/misc/firebird.conf
View File

@ -1019,6 +1019,19 @@
#GCPolicy = combined
# ----------------------------
# Maximum statement cache size
#
# The maximum amount of RAM used to cache unused DSQL compiled statements.
# If set to 0 (zero), statement cache is disabled.
#
# Per-database configurable.
#
# Type: integer
#
#MaxStatementCacheSize = 2M
# ----------------------------
# Security database
#

2
builds/win32/msvc15/engine.vcxproj
View File

@ -42,6 +42,7 @@
<ClCompile Include="..\..\..\src\dsql\DsqlCursor.cpp" />
<ClCompile Include="..\..\..\src\dsql\DSqlDataTypeUtil.cpp" />
<ClCompile Include="..\..\..\src\dsql\DsqlRequests.cpp" />
<ClCompile Include="..\..\..\src\dsql\DsqlStatementCache.cpp" />
<ClCompile Include="..\..\..\src\dsql\DsqlStatements.cpp" />
<ClCompile Include="..\..\..\src\dsql\errd.cpp" />
<ClCompile Include="..\..\..\src\dsql\ExprNodes.cpp" />
@ -190,6 +191,7 @@
<ClInclude Include="..\..\..\src\dsql\DsqlCursor.h" />
<ClInclude Include="..\..\..\src\dsql\DSqlDataTypeUtil.h" />
<ClInclude Include="..\..\..\src\dsql\DsqlRequests.h" />
<ClInclude Include="..\..\..\src\dsql\DsqlStatementCache.h" />
<ClInclude Include="..\..\..\src\dsql\DsqlStatements.h" />
<ClInclude Include="..\..\..\src\dsql\dsql_proto.h" />
<ClInclude Include="..\..\..\src\dsql\errd_proto.h" />

6
builds/win32/msvc15/engine.vcxproj.filters
View File

@ -144,6 +144,9 @@
<ClCompile Include="..\..\..\src\dsql\DsqlRequests.cpp">
<Filter>DSQL</Filter>
</ClCompile>
<ClCompile Include="..\..\..\src\dsql\DsqlStatementCache.cpp">
<Filter>DSQL</Filter>
</ClCompile>
<ClCompile Include="..\..\..\src\dsql\DsqlStatements.cpp">
<Filter>DSQL</Filter>
</ClCompile>
@ -560,6 +563,9 @@
<ClInclude Include="..\..\..\src\dsql\DsqlRequests.h">
<Filter>Header files</Filter>
</ClInclude>
<ClInclude Include="..\..\..\src\dsql\DsqlStatementCache.h">
<Filter>Header files</Filter>
</ClInclude>
<ClInclude Include="..\..\..\src\dsql\DsqlStatements.h">
<Filter>Header files</Filter>
</ClInclude>

10
src/common/classes/alloc.cpp
View File

@ -1850,6 +1850,11 @@ public:
// Create memory pool instance
static MemPool* createPool(MemPool* parent, MemoryStats& stats);
MemoryStats& getStatsGroup() noexcept
{
return *stats;
}
// Set statistics group for pool. Usage counters will be decremented from
// previously set group and added to new
void setStatsGroup(MemoryStats& stats) noexcept;
@ -2262,6 +2267,11 @@ void MemPool::setStatsGroup(MemoryStats& newStats) noexcept
stats->increment_usage(sav_used_memory);
}
MemoryStats& MemoryPool::getStatsGroup() noexcept
{
return pool->getStatsGroup();
}
void MemoryPool::setStatsGroup(MemoryStats& newStats) noexcept
{
pool->setStatsGroup(newStats);

2
src/common/classes/alloc.h
View File

@ -213,6 +213,8 @@ public:
// Get context pool for current thread of execution
static MemoryPool* getContextPool();
MemoryStats& getStatsGroup() noexcept;
// Set statistics group for pool. Usage counters will be decremented from
// previously set group and added to new
void setStatsGroup(MemoryStats& stats) noexcept;

2
src/common/config/config.cpp
View File

@ -410,6 +410,8 @@ void Config::checkValues()
checkIntForHiBound(KEY_TIP_CACHE_BLOCK_SIZE, MAX_ULONG, true);
checkIntForLoBound(KEY_INLINE_SORT_THRESHOLD, 0, true);
checkIntForLoBound(KEY_MAX_STATEMENT_CACHE_SIZE, 0, true);
}

6
src/common/config/config.h
View File

@ -188,6 +188,7 @@ enum ConfigKey
KEY_USE_FILESYSTEM_CACHE,
KEY_INLINE_SORT_THRESHOLD,
KEY_TEMP_PAGESPACE_DIR,
KEY_MAX_STATEMENT_CACHE_SIZE,
MAX_CONFIG_KEY // keep it last
};
@ -302,7 +303,8 @@ constexpr ConfigEntry entries[MAX_CONFIG_KEY] =
{TYPE_STRING, "DataTypeCompatibility", false, nullptr},
{TYPE_BOOLEAN, "UseFileSystemCache", false, true},
{TYPE_INTEGER, "InlineSortThreshold", false, 1000}, // bytes
{TYPE_STRING, "TempTableDirectory", false, ""}
{TYPE_STRING, "TempTableDirectory", false, ""},
{TYPE_INTEGER, "MaxStatementCacheSize", false, 2 * 1048576} // bytes
};
@ -624,6 +626,8 @@ public:
CONFIG_GET_PER_DB_KEY(ULONG, getInlineSortThreshold, KEY_INLINE_SORT_THRESHOLD, getInt);
CONFIG_GET_PER_DB_STR(getTempPageSpaceDirectory, KEY_TEMP_PAGESPACE_DIR);
CONFIG_GET_PER_DB_INT(getMaxStatementCacheSize, KEY_MAX_STATEMENT_CACHE_SIZE);
};
// Implementation of interface to access master configuration file

3
src/dsql/DsqlRequests.cpp
View File

@ -23,6 +23,7 @@
#include "../dsql/DsqlRequests.h"
#include "../dsql/dsql.h"
#include "../dsql/DsqlBatch.h"
#include "../dsql/DsqlStatementCache.h"
#include "../dsql/Nodes.h"
#include "../jrd/Statement.h"
#include "../jrd/req.h"
@ -1035,6 +1036,8 @@ void DsqlDdlRequest::execute(thread_db* tdbb, jrd_tra** traHandle,
{
AutoSetRestoreFlag<ULONG> execDdl(&tdbb->tdbb_flags, TDBB_repl_in_progress, true);
req_dbb->dbb_attachment->att_dsql_instance->dbb_statement_cache->purgeAllAttachments(tdbb);
node->executeDdl(tdbb, internalScratch, req_transaction);
const bool isInternalRequest =

10
src/dsql/DsqlRequests.h
View File

@ -75,11 +75,6 @@ public:
return nullptr;
}
virtual bool isDml() const
{
return false;
}
virtual DsqlCursor* openCursor(thread_db* tdbb, jrd_tra** traHandle,
Firebird::IMessageMetadata* inMeta, const UCHAR* inMsg,
Firebird::IMessageMetadata* outMeta, ULONG flags)
@ -165,11 +160,6 @@ public:
return request;
}
bool isDml() const override
{
return true;
}
DsqlCursor* openCursor(thread_db* tdbb, jrd_tra** traHandle,
Firebird::IMessageMetadata* inMeta, const UCHAR* inMsg,
Firebird::IMessageMetadata* outMeta, ULONG flags) override;

285
src/dsql/DsqlStatementCache.cpp Normal file
View File

@ -0,0 +1,285 @@
/*
* The contents of this file are subject to the Initial
* Developer's Public License Version 1.0 (the "License");
* you may not use this file except in compliance with the
* License. You may obtain a copy of the License at
* http://www.ibphoenix.com/main.nfs?a=ibphoenix&page=ibp_idpl.
*
* Software distributed under the License is distributed AS IS,
* WITHOUT WARRANTY OF ANY KIND, either express or implied.
* See the License for the specific language governing rights
* and limitations under the License.
*
* The Original Code was created by Adriano dos Santos Fernandes
* for the Firebird Open Source RDBMS project.
*
* Copyright (c) 2022 Adriano dos Santos Fernandes <adrianosf@gmail.com>
* and all contributors signed below.
*
* All Rights Reserved.
* Contributor(s): ______________________________________.
*/
#include "firebird.h"
#include "../dsql/DsqlStatementCache.h"
#include "../dsql/DsqlStatements.h"
#include "../jrd/Attachment.h"
#include "../jrd/Statement.h"
#include "../jrd/lck.h"
#include "../jrd/lck_proto.h"
using namespace Firebird;
using namespace Jrd;
// Class DsqlStatementCache
DsqlStatementCache::DsqlStatementCache(MemoryPool& o, Attachment* attachment)
: PermanentStorage(o),
map(o),
activeStatementList(o),
inactiveStatementList(o)
{
const auto dbb = attachment->att_database;
maxCacheSize = dbb->dbb_config->getMaxStatementCacheSize();
}
DsqlStatementCache::~DsqlStatementCache()
{
purge(JRD_get_thread_data());
}
int DsqlStatementCache::blockingAst(void* astObject)
{
#ifdef DSQL_STATEMENT_CACHE_DEBUG
printf("DsqlStatementCache::blockingAst()\n");
#endif
const auto self = static_cast<DsqlStatementCache*>(astObject);
try
{
const auto dbb = self->lock->lck_dbb;
AsyncContextHolder tdbb(dbb, FB_FUNCTION, self->lock);
self->purge(tdbb);
}
catch (const Exception&)
{} // no-op
return 0;
}
RefPtr<DsqlStatement> DsqlStatementCache::getStatement(thread_db* tdbb, const string& text, USHORT clientDialect,
bool isInternalRequest)
{
RefStrPtr key;
buildStatementKey(tdbb, key, text, clientDialect, isInternalRequest);
if (const auto entryPtr = map.get(key))
{
const auto entry = *entryPtr;
auto dsqlStatement(entry->dsqlStatement);
string verifyKey;
buildVerifyKey(tdbb, verifyKey, isInternalRequest);
FB_SIZE_T verifyPos;
if (!entry->verifyCache.find(verifyKey, verifyPos))
{
dsqlStatement->getStatement()->verifyAccess(tdbb);
entry->verifyCache.insert(verifyPos, verifyKey);
}
if (!entry->active)
{
entry->dsqlStatement->setCacheKey(key);
// Active statement has cacheKey and will tell us when it's going to be released.
entry->dsqlStatement->release();
entry->active = true;
cacheSize -= entry->size;
activeStatementList.splice(activeStatementList.end(), inactiveStatementList, entry);
}
#ifdef DSQL_STATEMENT_CACHE_DEBUG
dump();
#endif
return dsqlStatement;
}
return {};
}
void DsqlStatementCache::putStatement(thread_db* tdbb, const string& text, USHORT clientDialect,
bool isInternalRequest, RefPtr<DsqlStatement> dsqlStatement)
{
fb_assert(dsqlStatement->isDml());
const unsigned statementSize = dsqlStatement->getSize();
RefStrPtr key;
buildStatementKey(tdbb, key, text, clientDialect, isInternalRequest);
StatementEntry newStatement(getPool());
newStatement.key = key;
newStatement.size = statementSize;
newStatement.dsqlStatement = std::move(dsqlStatement);
newStatement.active = true;
string verifyKey;
buildVerifyKey(tdbb, verifyKey, isInternalRequest);
newStatement.verifyCache.add(verifyKey);
newStatement.dsqlStatement->setCacheKey(key);
// Active statement has cacheKey and will tell us when it's going to be released.
newStatement.dsqlStatement->release();
activeStatementList.pushBack(std::move(newStatement));
map.put(key, --activeStatementList.end());
if (!lock)
{
lock = FB_NEW_RPT(getPool(), 0) Lock(tdbb, 0, LCK_dsql_statement_cache, this, blockingAst);
LCK_lock(tdbb, lock, LCK_SR, LCK_WAIT);
}
#ifdef DSQL_STATEMENT_CACHE_DEBUG
dump();
#endif
}
void DsqlStatementCache::statementGoingInactive(Firebird::RefStrPtr& key)
{
const auto entryPtr = map.get(key);
if (!entryPtr)
{
fb_assert(false);
return;
}
const auto entry = *entryPtr;
fb_assert(entry->active);
entry->active = false;
entry->size = entry->dsqlStatement->getSize(); // update size
inactiveStatementList.splice(inactiveStatementList.end(), activeStatementList, entry);
cacheSize += entry->size;
if (cacheSize > maxCacheSize)
shrink();
}
void DsqlStatementCache::purge(thread_db* tdbb)
{
for (auto& entry : activeStatementList)
{
entry.dsqlStatement->addRef();
entry.dsqlStatement->resetCacheKey();
}
map.clear();
activeStatementList.clear();
inactiveStatementList.clear();
cacheSize = 0;
if (lock)
{
LCK_release(tdbb, lock);
lock.reset();
}
}
void DsqlStatementCache::purgeAllAttachments(thread_db* tdbb)
{
if (lock)
LCK_convert(tdbb, lock, LCK_EX, LCK_WAIT);
else
{
lock = FB_NEW_RPT(getPool(), 0) Lock(tdbb, 0, LCK_dsql_statement_cache, this, blockingAst);
LCK_lock(tdbb, lock, LCK_EX, LCK_WAIT);
}
purge(tdbb);
}
void DsqlStatementCache::buildStatementKey(thread_db* tdbb, RefStrPtr& key, const string& text, USHORT clientDialect,
bool isInternalRequest)
{
const auto attachment = tdbb->getAttachment();
const SSHORT charSetId = isInternalRequest ? CS_METADATA : attachment->att_charset;
key = FB_NEW_POOL(getPool()) RefString(getPool());
key->resize(1 + sizeof(charSetId) + text.length());
char* p = key->begin();
*p = (clientDialect << 1) | int(isInternalRequest);
memcpy(p + 1, &charSetId, sizeof(charSetId));
memcpy(p + 1 + sizeof(charSetId), text.c_str(), text.length());
}
void DsqlStatementCache::buildVerifyKey(thread_db* tdbb, string& key, bool isInternalRequest)
{
key.clear();
const auto attachment = tdbb->getAttachment();
if (isInternalRequest || !attachment->att_user)
return;
const auto& roles = attachment->att_user->getGrantedRoles(tdbb);
string roleStr;
for (const auto& role : roles)
{
roleStr.printf("%d,%s,", int(role.length()), role.c_str());
key += roleStr;
}
}
void DsqlStatementCache::shrink()
{
#ifdef DSQL_STATEMENT_CACHE_DEBUG
printf("DsqlStatementCache::shrink() - cacheSize: %u, maxCacheSize: %u\n\n", cacheSize, maxCacheSize);
#endif
while (cacheSize > maxCacheSize && !inactiveStatementList.isEmpty())
{
const auto& front = inactiveStatementList.front();
map.remove(front.key);
cacheSize -= front.size;
inactiveStatementList.erase(inactiveStatementList.begin());
}
#ifdef DSQL_STATEMENT_CACHE_DEBUG
dump();
#endif
}
#ifdef DSQL_STATEMENT_CACHE_DEBUG
void DsqlStatementCache::dump()
{
printf("DsqlStatementCache::dump() - cacheSize: %u, maxCacheSize: %u\n\n", cacheSize, maxCacheSize);
printf("\tactive:\n");
for (auto& entry : activeStatementList)
printf("\t\tsize: %u; text: %s\n", entry.size, entry.dsqlStatement->getSqlText()->c_str());
printf("\n\tinactive:\n");
for (auto& entry : inactiveStatementList)
printf("\t\tsize: %u; text: %s\n", entry.size, entry.dsqlStatement->getSqlText()->c_str());
printf("\n");
}
#endif

137
src/dsql/DsqlStatementCache.h Normal file
View File

@ -0,0 +1,137 @@
/*
* The contents of this file are subject to the Initial
* Developer's Public License Version 1.0 (the "License");
* you may not use this file except in compliance with the
* License. You may obtain a copy of the License at
* http://www.ibphoenix.com/main.nfs?a=ibphoenix&page=ibp_idpl.
*
* Software distributed under the License is distributed AS IS,
* WITHOUT WARRANTY OF ANY KIND, either express or implied.
* See the License for the specific language governing rights
* and limitations under the License.
*
* The Original Code was created by Adriano dos Santos Fernandes
* for the Firebird Open Source RDBMS project.
*
* Copyright (c) 2022 Adriano dos Santos Fernandes <adrianosf@gmail.com>
* and all contributors signed below.
*
* All Rights Reserved.
* Contributor(s): ______________________________________.
*/
#ifndef DSQL_STATEMENT_CACHE_H
#define DSQL_STATEMENT_CACHE_H
///#define DSQL_STATEMENT_CACHE_DEBUG 1
#include "../common/classes/alloc.h"
#include "../common/classes/DoublyLinkedList.h"
#include "../common/classes/fb_string.h"
#include "../common/classes/GenericMap.h"
#include "../common/classes/objects_array.h"
#include "../common/classes/RefCounted.h"
namespace Jrd {
class Attachment;
class DsqlStatement;
class Lock;
class thread_db;
class DsqlStatementCache final : public Firebird::PermanentStorage
{
private:
struct StatementEntry
{
explicit StatementEntry(MemoryPool& p)
: verifyCache(p)
{
}
StatementEntry(MemoryPool& p, StatementEntry&& o)
: key(std::move(o.key)),
dsqlStatement(std::move(o.dsqlStatement)),
verifyCache(p, std::move(o.verifyCache)),
size(o.size),
active(o.active)
{
}
StatementEntry(const StatementEntry&) = delete;
StatementEntry& operator=(const StatementEntry&) = delete;
Firebird::RefStrPtr key;
Firebird::RefPtr<DsqlStatement> dsqlStatement;
Firebird::SortedObjectsArray<Firebird::string> verifyCache;
unsigned size = 0;
bool active = true;
};
class RefStrPtrComparator
{
public:
static bool greaterThan(const Firebird::RefStrPtr& i1, const Firebird::RefStrPtr& i2)
{
return *i1 > *i2;
}
};
public:
explicit DsqlStatementCache(MemoryPool& o, Attachment* attachment);
~DsqlStatementCache();
DsqlStatementCache(const DsqlStatementCache&) = delete;
DsqlStatementCache& operator=(const DsqlStatementCache&) = delete;
private:
static int blockingAst(void* astObject);
public:
bool isActive() const
{
return maxCacheSize > 0;
}
Firebird::RefPtr<DsqlStatement> getStatement(thread_db* tdbb, const Firebird::string& text,
USHORT clientDialect, bool isInternalRequest);
void putStatement(thread_db* tdbb, const Firebird::string& text, USHORT clientDialect, bool isInternalRequest,
Firebird::RefPtr<DsqlStatement> dsqlStatement);
void statementGoingInactive(Firebird::RefStrPtr& key);
void purge(thread_db* tdbb);
void purgeAllAttachments(thread_db* tdbb);
private:
void buildStatementKey(thread_db* tdbb, Firebird::RefStrPtr& key, const Firebird::string& text,
USHORT clientDialect, bool isInternalRequest);
void buildVerifyKey(thread_db* tdbb, Firebird::string& key, bool isInternalRequest);
void shrink();
#ifdef DSQL_STATEMENT_CACHE_DEBUG
void dump();
#endif
private:
Firebird::NonPooledMap<
Firebird::RefStrPtr,
Firebird::DoublyLinkedList<StatementEntry>::Iterator,
RefStrPtrComparator
> map;
Firebird::DoublyLinkedList<StatementEntry> activeStatementList;
Firebird::DoublyLinkedList<StatementEntry> inactiveStatementList;
Firebird::AutoPtr<Lock> lock;
unsigned maxCacheSize = 0;
unsigned cacheSize = 0;
};
} // namespace Jrd
#endif // DSQL_STATEMENT_CACHE_H

22
src/dsql/DsqlStatements.cpp
View File

@ -24,6 +24,7 @@
#include "../dsql/dsql.h"
#include "../dsql/Nodes.h"
#include "../dsql/DsqlCompilerScratch.h"
#include "../dsql/DsqlStatementCache.h"
#include "../jrd/Statement.h"
#include "../dsql/errd_proto.h"
#include "../dsql/gen_proto.h"
@ -58,12 +59,22 @@ void DsqlStatement::rethrowDdlException(status_exception& ex, bool metadataUpdat
int DsqlStatement::release()
{
fb_assert(refCounter.value() > 0);
const int refCnt = --refCounter;
int refCnt = --refCounter;
if (!refCnt)
{
doRelease();
dsqlAttachment->deletePool(&getPool());
if (cacheKey)
{
refCnt = ++refCounter;
auto key = cacheKey;
cacheKey = nullptr;
dsqlAttachment->dbb_statement_cache->statementGoingInactive(key);
}
else
{
doRelease();
dsqlAttachment->deletePool(&getPool());
}
}
return refCnt;
@ -119,6 +130,11 @@ void DsqlDmlStatement::doRelease()
DsqlStatement::doRelease();
}
unsigned DsqlDmlStatement::getSize() const
{
return DsqlStatement::getSize() + statement->getSize();
}
void DsqlDmlStatement::dsqlPass(thread_db* tdbb, DsqlCompilerScratch* scratch, ntrace_result_t* traceResult)
{
{ // scope

30
src/dsql/DsqlStatements.h
View File

@ -26,6 +26,7 @@
#include "../common/classes/array.h"
#include "../common/classes/fb_string.h"
#include "../common/classes/NestConst.h"
#include "../common/classes/RefCounted.h"
#include "../jrd/jrd.h"
#include "../jrd/ntrace.h"
#include "../dsql/DsqlRequests.h"
@ -67,14 +68,15 @@ public:
static void rethrowDdlException(Firebird::status_exception& ex, bool metadataUpdate, DdlNode* node);
public:
DsqlStatement(MemoryPool& p, dsql_dbb* aDsqlAttachment)
: PermanentStorage(p),
DsqlStatement(MemoryPool& pool, dsql_dbb* aDsqlAttachment)
: PermanentStorage(pool),
dsqlAttachment(aDsqlAttachment),
type(TYPE_SELECT),
flags(0),
blrVersion(5),
ports(p)
ports(pool)
{
pool.setStatsGroup(memoryStats);
}
protected:
@ -133,7 +135,15 @@ public:
const dsql_par* getEof() const { return eof; }
void setEof(dsql_par* value) { eof = value; }
void setCacheKey(Firebird::RefStrPtr& value) { cacheKey = value; }
void resetCacheKey() { cacheKey = nullptr; }
public:
virtual bool isDml() const
{
return false;
}
virtual Statement* getStatement() const
{
return nullptr;
@ -149,6 +159,11 @@ public:
return true;
}
virtual unsigned getSize() const
{
return (unsigned) memoryStats.getCurrentUsage();
}
virtual void dsqlPass(thread_db* tdbb, DsqlCompilerScratch* scratch, ntrace_result_t* traceResult) = 0;
virtual DsqlRequest* createRequest(thread_db* tdbb, dsql_dbb* dbb) = 0;
@ -157,11 +172,13 @@ protected:
protected:
dsql_dbb* dsqlAttachment;
Firebird::MemoryStats memoryStats;
Type type; // Type of statement
ULONG flags; // generic flag
unsigned blrVersion;
Firebird::RefStrPtr sqlText;
Firebird::RefStrPtr orgText;
Firebird::RefStrPtr cacheKey;
Firebird::Array<dsql_msg*> ports; // Port messages
dsql_msg* sendMsg = nullptr; // Message to be sent to start request
dsql_msg* receiveMsg = nullptr; // Per record message to be received
@ -182,11 +199,18 @@ public:
}
public:
bool isDml() const override
{
return true;
}
Statement* getStatement() const override
{
return statement;
}
unsigned getSize() const override;
void dsqlPass(thread_db* tdbb, DsqlCompilerScratch* scratch, ntrace_result_t* traceResult) override;
DsqlDmlRequest* createRequest(thread_db* tdbb, dsql_dbb* dbb) override;

39
src/dsql/dsql.cpp
View File

@ -72,6 +72,7 @@
#include "../common/utils_proto.h"
#include "../common/StatusArg.h"
#include "../dsql/DsqlBatch.h"
#include "../dsql/DsqlStatementCache.h"
#ifdef HAVE_CTYPE_H
#include <ctype.h>
@ -109,6 +110,21 @@ namespace
IMPLEMENT_TRACE_ROUTINE(dsql_trace, "DSQL")
#endif
dsql_dbb::dsql_dbb(MemoryPool& p, Attachment* attachment)
: dbb_relations(p),
dbb_procedures(p),
dbb_functions(p),
dbb_charsets(p),
dbb_collations(p),
dbb_charsets_by_id(p),
dbb_cursors(p),
dbb_pool(p),
dbb_dfl_charset(p)
{
dbb_attachment = attachment;
dbb_statement_cache = FB_NEW_POOL(p) DsqlStatementCache(p, dbb_attachment);
}
dsql_dbb::~dsql_dbb()
{
}
@ -411,8 +427,7 @@ static dsql_dbb* init(thread_db* tdbb, Jrd::Attachment* attachment)
return attachment->att_dsql_instance;
MemoryPool& pool = *attachment->createPool();
dsql_dbb* const database = FB_NEW_POOL(pool) dsql_dbb(pool);
database->dbb_attachment = attachment;
dsql_dbb* const database = FB_NEW_POOL(pool) dsql_dbb(pool, attachment);
attachment->att_dsql_instance = database;
INI_init_dsql(tdbb, database);
@ -497,12 +512,24 @@ static RefPtr<DsqlStatement> prepareStatement(thread_db* tdbb, dsql_dbb* databas
Arg::Gds(isc_sql_too_long) << Arg::Num(MAX_SQL_LENGTH));
}
string textStr(text, textLength);
const bool isStatementCacheActive = database->dbb_statement_cache->isActive();
RefPtr<DsqlStatement> dsqlStatement;
if (isStatementCacheActive)
{
dsqlStatement = database->dbb_statement_cache->getStatement(tdbb, textStr, clientDialect, isInternalRequest);
if (dsqlStatement)
return dsqlStatement;
}
// allocate the statement block, then prepare the statement
MemoryPool* scratchPool = nullptr;
DsqlCompilerScratch* scratch = nullptr;
MemoryPool* statementPool = database->createPool();
RefPtr<DsqlStatement> dsqlStatement;
Jrd::ContextPoolHolder statementContext(tdbb, statementPool);
try
@ -593,6 +620,12 @@ static RefPtr<DsqlStatement> prepareStatement(thread_db* tdbb, dsql_dbb* databas
if (!isInternalRequest && dsqlStatement->mustBeReplicated())
dsqlStatement->setOrgText(text, textLength);
if (isStatementCacheActive && dsqlStatement->isDml())
{
database->dbb_statement_cache->putStatement(tdbb,
textStr, clientDialect, isInternalRequest, dsqlStatement);
}
return dsqlStatement;
}
catch (const Exception&)

22
src/dsql/dsql.h
View File

@ -77,12 +77,9 @@ namespace Jrd
class Attachment;
class Database;
class DsqlCompilerScratch;
class DsqlDmlStatement;
class DdlNode;
class DsqlStatement;
class DsqlStatementCache;
class RseNode;
class StmtNode;
class TransactionNode;
class SessionManagementNode;
class ValueExprNode;
class ValueListNode;
class WindowClause;
@ -92,7 +89,6 @@ namespace Jrd
struct bid;
class dsql_ctx;
class dsql_msg;
class dsql_par;
class dsql_map;
class dsql_intlsym;
@ -130,24 +126,14 @@ public:
Firebird::LeftPooledMap<MetaName, class dsql_intlsym*> dbb_collations; // known collations in database
Firebird::NonPooledMap<SSHORT, dsql_intlsym*> dbb_charsets_by_id; // charsets sorted by charset_id
Firebird::LeftPooledMap<Firebird::string, DsqlDmlRequest*> dbb_cursors; // known cursors in database
Firebird::AutoPtr<DsqlStatementCache> dbb_statement_cache;
MemoryPool& dbb_pool; // The current pool for the dbb
Attachment* dbb_attachment;
MetaName dbb_dfl_charset;
bool dbb_no_charset;
explicit dsql_dbb(MemoryPool& p)
: dbb_relations(p),
dbb_procedures(p),
dbb_functions(p),
dbb_charsets(p),
dbb_collations(p),
dbb_charsets_by_id(p),
dbb_cursors(p),
dbb_pool(p),
dbb_dfl_charset(p)
{}
dsql_dbb(MemoryPool& p, Attachment* attachment);
~dsql_dbb();
MemoryPool* createPool()

6
src/jrd/Attachment.cpp
View File

@ -145,6 +145,8 @@ void Jrd::Attachment::destroy(Attachment* const attachment)
MemoryPool* Jrd::Attachment::createPool()
{
MemoryPool* const pool = MemoryPool::createPool(att_pool, att_memory_stats);
auto stats = FB_NEW_POOL(*pool) MemoryStats(&att_memory_stats);
pool->setStatsGroup(*stats);
att_pools.add(pool);
return pool;
}
@ -154,9 +156,7 @@ void Jrd::Attachment::deletePool(MemoryPool* pool)
{
if (pool)
{
FB_SIZE_T pos;
if (att_pools.find(pool, pos))
att_pools.remove(pos);
att_pools.findAndRemove(pool);
#ifdef DEBUG_LCK_LIST
// hvlad: this could be slow, use only when absolutely necessary

6
src/jrd/Monitoring.cpp
View File

@ -1172,7 +1172,13 @@ void Monitoring::putStatement(SnapshotData::DumpRecord& record, const Statement*
record.storeInteger(f_mon_cmp_stmt_type, obj_trigger);
}
// statistics
const int stat_id = fb_utils::genUniqueId();
record.storeGlobalId(f_mon_cmp_stmt_stat_id, getGlobalId(stat_id));
record.write();
putMemoryUsage(record, statement->pool->getStatsGroup(), stat_id, stat_cmp_statement);
}

126
src/jrd/Statement.cpp
View File

@ -74,8 +74,7 @@ Statement::Statement(thread_db* tdbb, MemoryPool* p, CompilerScratch* csb)
localTables(*p),
invariants(*p),
blr(*p),
mapFieldInfo(*p),
mapItemInfo(*p)
mapFieldInfo(*p)
{
try
{
@ -86,9 +85,16 @@ Statement::Statement(thread_db* tdbb, MemoryPool* p, CompilerScratch* csb)
static_cast<StmtNode*>(csb->csb_node) : NULL;
accessList = csb->csb_access;
csb->csb_access.clear();
externalList = csb->csb_external;
csb->csb_external.clear();
mapFieldInfo.takeOwnership(csb->csb_map_field_info);
resources = csb->csb_resources; // Assign array contents
csb->csb_resources.clear();
impureSize = csb->csb_impure;
//if (csb->csb_g_flags & csb_blr_version4)
@ -155,19 +161,22 @@ Statement::Statement(thread_db* tdbb, MemoryPool* p, CompilerScratch* csb)
// make a vector of all used RSEs
fors = csb->csb_fors;
csb->csb_fors.clear();
localTables = csb->csb_localTables;
csb->csb_localTables.clear();
// make a vector of all invariant-type nodes, so that we will
// be able to easily reinitialize them when we restart the request
invariants.join(csb->csb_invariants);
csb->csb_invariants.clear();
rpbsSetup.grow(csb->csb_n_stream);
CompilerScratch::csb_repeat* tail = csb->csb_rpt.begin();
const CompilerScratch::csb_repeat* const streams_end = tail + csb->csb_n_stream;
auto tail = csb->csb_rpt.begin();
const auto* const streams_end = tail + csb->csb_n_stream;
for (record_param* rpb = rpbsSetup.begin(); tail < streams_end; ++rpb, ++tail)
for (auto rpb = rpbsSetup.begin(); tail < streams_end; ++rpb, ++tail)
{
// fetch input stream for update if all booleans matched against indices
if ((tail->csb_flags & csb_update) && !(tail->csb_flags & csb_unmatched))
@ -186,6 +195,22 @@ Statement::Statement(thread_db* tdbb, MemoryPool* p, CompilerScratch* csb)
delete tail->csb_fields;
tail->csb_fields = NULL;
}
if (csb->csb_variables)
csb->csb_variables->clear();
csb->csb_current_nodes.free();
csb->csb_current_for_nodes.free();
csb->csb_computing_fields.free();
csb->csb_variables_used_in_subroutines.free();
csb->csb_dbg_info.reset();
csb->csb_map_item_info.clear();
csb->csb_message_pad.clear();
csb->subFunctions.clear();
csb->subProcedures.clear();
csb->outerMessagesMap.clear();
csb->outerVarsMap.clear();
csb->csb_rpt.free();
}
catch (Exception&)
{
@ -206,13 +231,15 @@ Statement* Statement::makeStatement(thread_db* tdbb, CompilerScratch* csb, bool
DEV_BLKCHK(csb, type_csb);
SET_TDBB(tdbb);
Database* const dbb = tdbb->getDatabase();
const auto dbb = tdbb->getDatabase();
fb_assert(dbb);
Request* const old_request = tdbb->getRequest();
tdbb->setRequest(NULL);
const auto attachment = tdbb->getAttachment();
Statement* statement = NULL;
const auto old_request = tdbb->getRequest();
tdbb->setRequest(nullptr);
Statement* statement = nullptr;
try
{
@ -277,7 +304,7 @@ Statement* Statement::makeStatement(thread_db* tdbb, CompilerScratch* csb, bool
// Build the statement and the final request block.
MemoryPool* const pool = tdbb->getDefaultPool();
const auto pool = tdbb->getDefaultPool();
statement = FB_NEW_POOL(*pool) Statement(tdbb, pool, csb);
@ -288,12 +315,8 @@ Statement* Statement::makeStatement(thread_db* tdbb, CompilerScratch* csb, bool
if (statement)
{
// Release sub statements.
for (Statement** subStatement = statement->subStatements.begin();
subStatement != statement->subStatements.end();
++subStatement)
{
(*subStatement)->release(tdbb);
}
for (auto subStatement : statement->subStatements)
subStatement->release(tdbb);
}
ex.stuffException(tdbb->tdbb_status_vector);
@ -302,9 +325,14 @@ Statement* Statement::makeStatement(thread_db* tdbb, CompilerScratch* csb, bool
}
if (internalFlag)
{
statement->flags |= FLAG_INTERNAL;
statement->charSetId = CS_METADATA;
}
else
statement->charSetId = attachment->att_charset;
tdbb->getAttachment()->att_statements.add(statement);
attachment->att_statements.add(statement);
return statement;
}
@ -402,17 +430,11 @@ Request* Statement::getRequest(thread_db* tdbb, USHORT level)
if (level < requests.getCount() && requests[level])
return requests[level];
requests.grow(level + 1);
MemoryStats* const parentStats = (flags & FLAG_INTERNAL) ?
&dbb->dbb_memory_stats : &attachment->att_memory_stats;
// Create the request.
Request* const request = FB_NEW_POOL(*pool) Request(attachment, this, parentStats);
if (level == 0)
pool->setStatsGroup(request->req_memory_stats);
AutoMemoryPool reqPool(MemoryPool::createPool(pool));
const auto request = FB_NEW_POOL(*reqPool) Request(reqPool, attachment, this);
requests.grow(level + 1);
requests[level] = request;
return request;
@ -500,15 +522,13 @@ void Statement::verifyAccess(thread_db* tdbb)
if (!routine->getStatement())
continue;
for (const AccessItem* access = routine->getStatement()->accessList.begin();
access != routine->getStatement()->accessList.end();
++access)
for (const auto& access : routine->getStatement()->accessList)
{
MetaName userName = item->user;
if (access->acc_ss_rel_id)
if (access.acc_ss_rel_id)
{
const jrd_rel* view = MET_lookup_relation_id(tdbb, access->acc_ss_rel_id, false);
const jrd_rel* view = MET_lookup_relation_id(tdbb, access.acc_ss_rel_id, false);
if (view && (view->rel_flags & REL_sql_relation))
userName = view->rel_owner_name;
}
@ -517,17 +537,17 @@ void Statement::verifyAccess(thread_db* tdbb)
UserId* effectiveUser = userName.hasData() ? attachment->getUserId(userName) : attachment->att_ss_user;
AutoSetRestore<UserId*> userIdHolder(&attachment->att_ss_user, effectiveUser);
const SecurityClass* sec_class = SCL_get_class(tdbb, access->acc_security_name.c_str());
const SecurityClass* sec_class = SCL_get_class(tdbb, access.acc_security_name.c_str());
if (routine->getName().package.isEmpty())
{
SCL_check_access(tdbb, sec_class, aclType, routine->getName().identifier,
access->acc_mask, access->acc_type, true, access->acc_name, access->acc_r_name);
access.acc_mask, access.acc_type, true, access.acc_name, access.acc_r_name);
}
else
{
SCL_check_access(tdbb, sec_class, id_package, routine->getName().package,
access->acc_mask, access->acc_type, true, access->acc_name, access->acc_r_name);
access.acc_mask, access.acc_type, true, access.acc_name, access.acc_r_name);
}
}
}
@ -650,7 +670,11 @@ void Statement::release(thread_db* tdbb)
}
for (Request** instance = requests.begin(); instance != requests.end(); ++instance)
{
EXE_release(tdbb, *instance);
MemoryPool::deletePool((*instance)->req_pool);
*instance = nullptr;
}
const auto attachment = tdbb->getAttachment();
@ -842,41 +866,33 @@ template <typename T> static void makeSubRoutines(thread_db* tdbb, Statement* st
{
typename T::Accessor subAccessor(&subs);
for (bool found = subAccessor.getFirst(); found; found = subAccessor.getNext())
for (auto& sub : subs)
{
typename T::ValueType subNode = subAccessor.current()->second;
Routine* subRoutine = subNode->routine;
CompilerScratch*& subCsb = subNode->subCsb;
auto subNode = sub.second;
auto subRoutine = subNode->routine;
auto& subCsb = subNode->subCsb;
Statement* subStatement = Statement::makeStatement(tdbb, subCsb, false);
auto subStatement = Statement::makeStatement(tdbb, subCsb, false);
subStatement->parentStatement = statement;
subRoutine->setStatement(subStatement);
// Move dependencies and permissions from the sub routine to the parent.
for (CompilerScratch::Dependency* dependency = subCsb->csb_dependencies.begin();
dependency != subCsb->csb_dependencies.end();
++dependency)
{
csb->csb_dependencies.push(*dependency);
}
for (auto& dependency : subCsb->csb_dependencies)
csb->csb_dependencies.push(dependency);
for (ExternalAccess* access = subCsb->csb_external.begin();
access != subCsb->csb_external.end();
++access)
for (auto& access : subStatement->externalList)
{
FB_SIZE_T i;
if (!csb->csb_external.find(*access, i))
csb->csb_external.insert(i, *access);
if (!csb->csb_external.find(access, i))
csb->csb_external.insert(i, access);
}
for (AccessItem* access = subCsb->csb_access.begin();
access != subCsb->csb_access.end();
++access)
for (auto& access : subStatement->accessList)
{
FB_SIZE_T i;
if (!csb->csb_access.find(*access, i))
csb->csb_access.insert(i, *access);
if (!csb->csb_access.find(access, i))
csb->csb_access.insert(i, access);
}
delete subCsb;

7
src/jrd/Statement.h
View File

@ -55,6 +55,11 @@ public:
return id;
}
unsigned getSize() const
{
return (unsigned) pool->getStatsGroup().getCurrentUsage();
}
const Routine* getRoutine() const;
bool isActive() const;
@ -78,6 +83,7 @@ public:
unsigned blrVersion;
ULONG impureSize; // Size of impure area
mutable StmtNumber id; // statement identifier
USHORT charSetId; // client character set (CS_METADATA for internal statements)
Firebird::Array<record_param> rpbsSetup;
Firebird::Array<Request*> requests; // vector of requests
ExternalAccessList externalList; // Access to procedures/triggers to be checked
@ -96,7 +102,6 @@ public:
Firebird::RefStrPtr sqlText; // SQL text (encoded in the metadata charset)
Firebird::Array<UCHAR> blr; // BLR for non-SQL query
MapFieldInfo mapFieldInfo; // Map field name to field info
MapItemInfo mapItemInfo; // Map item to item info
};

3
src/jrd/constants.h
View File

@ -284,7 +284,8 @@ enum stat_group_t {
stat_attachment = 1,
stat_transaction = 2,
stat_statement = 3,
stat_call = 4
stat_call = 4,
stat_cmp_statement = 5
};
enum InfoType

19
src/jrd/jrd.cpp
View File

@ -134,6 +134,7 @@
#include "../dsql/dsql.h"
#include "../dsql/dsql_proto.h"
#include "../dsql/DsqlBatch.h"
#include "../dsql/DsqlStatementCache.h"
#ifdef WIN_NT
#include <process.h>
@ -4714,13 +4715,9 @@ void JAttachment::transactRequest(CheckStatusWrapper* user_status, ITransaction*
CompilerScratch* csb = PAR_parse(tdbb, reinterpret_cast<const UCHAR*>(blr),
blr_length, false);
request = Statement::makeRequest(tdbb, csb, false);
request->getStatement()->verifyAccess(tdbb);
for (FB_SIZE_T i = 0; i < csb->csb_rpt.getCount(); i++)
{
const MessageNode* node = csb->csb_rpt[i].csb_message;
if (node)
if (const auto node = csb->csb_rpt[i].csb_message)
{
if (node->messageNumber == 0)
inMessage = node;
@ -4728,6 +4725,9 @@ void JAttachment::transactRequest(CheckStatusWrapper* user_status, ITransaction*
outMessage = node;
}
}
request = Statement::makeRequest(tdbb, csb, false);
request->getStatement()->verifyAccess(tdbb);
}
catch (const Exception&)
{
@ -7551,6 +7551,9 @@ void release_attachment(thread_db* tdbb, Jrd::Attachment* attachment)
attachment->att_replicator = nullptr;
if (attachment->att_dsql_instance)
attachment->att_dsql_instance->dbb_statement_cache->purge(tdbb);
while (attachment->att_repl_appliers.hasData())
{
AutoPtr<Applier> cleanupApplier(attachment->att_repl_appliers.pop());
@ -8906,8 +8909,10 @@ void thread_db::setRequest(Request* val)
SSHORT thread_db::getCharSet() const
{
if (request && request->charSetId != CS_dynamic)
return request->charSetId;
USHORT charSetId;
if (request && (charSetId = request->getStatement()->charSetId) != CS_dynamic)
return charSetId;
return attachment->att_charset;
}

1
src/jrd/lck.cpp
View File

@ -584,6 +584,7 @@ static lck_owner_t get_owner_type(enum lck_t lock_type)
case LCK_record_gc:
case LCK_alter_database:
case LCK_repl_tables:
case LCK_dsql_statement_cache:
owner_type = LCK_OWNER_attachment;
break;

3
src/jrd/lck.h
View File

@ -74,7 +74,8 @@ enum lck_t {
LCK_record_gc, // Record-level GC lock
LCK_alter_database, // ALTER DATABASE lock
LCK_repl_state, // Replication state lock
LCK_repl_tables // Replication set lock
LCK_repl_tables, // Replication set lock
LCK_dsql_statement_cache // DSQL statement cache lock
};
// Lock owner types

1
src/jrd/relations.h
View File

@ -752,4 +752,5 @@ RELATION(nam_mon_compiled_statements, rel_mon_compiled_statements, ODS_13_1, rel
FIELD(f_mon_cmp_stmt_name, nam_mon_obj_name, fld_gnr_name, 0, ODS_13_1)
FIELD(f_mon_cmp_stmt_type, nam_mon_obj_type, fld_obj_type, 0, ODS_13_1)
FIELD(f_mon_cmp_stmt_pkg_name, nam_mon_pkg_name, fld_pkg_name, 0, ODS_13_1)
FIELD(f_mon_cmp_stmt_stat_id, nam_mon_stat_id, fld_stat_id, 0, ODS_13_1)
END_RELATION

15
src/jrd/req.h
View File

@ -266,11 +266,10 @@ private:
};
public:
Request(Attachment* attachment, /*const*/ Statement* aStatement,
Firebird::MemoryStats* parent_stats)
Request(Firebird::AutoMemoryPool& pool, Attachment* attachment, /*const*/ Statement* aStatement)
: statement(aStatement),
req_pool(statement->pool),
req_memory_stats(parent_stats),
req_pool(pool),
req_memory_stats(&aStatement->pool->getStatsGroup()),
req_blobs(req_pool),
req_stats(*req_pool),
req_base_stats(*req_pool),
@ -288,6 +287,9 @@ public:
setAttachment(attachment);
req_rpb = statement->rpbsSetup;
impureArea.grow(statement->impureSize);
pool->setStatsGroup(req_memory_stats);
pool.release();
}
Statement* getStatement()
@ -313,8 +315,6 @@ public:
void setAttachment(Attachment* newAttachment)
{
req_attachment = newAttachment;
charSetId = statement->flags & Statement::FLAG_INTERNAL ?
CS_METADATA : req_attachment->att_charset;
}
bool isRoot() const
@ -351,9 +351,9 @@ private:
public:
MemoryPool* req_pool;
Firebird::MemoryStats req_memory_stats;
Attachment* req_attachment; // database attachment
USHORT req_incarnation; // incarnation number
Firebird::MemoryStats req_memory_stats;
// Transaction pointer and doubly linked list pointers for requests in this
// transaction. Maintained by TRA_attach_request/TRA_detach_request.
@ -399,7 +399,6 @@ public:
SortOwner req_sorts;
Firebird::Array<record_param> req_rpb; // record parameter blocks
Firebird::Array<UCHAR> impureArea; // impure area
USHORT charSetId; // "client" character set of the request
TriggerAction req_trigger_action; // action that caused trigger to fire
// Fields to support read consistency in READ COMMITTED transactions

7
src/jrd/scl.h
View File

@ -366,6 +366,13 @@ public:
return usr_granted_roles.exist(role);
}
const auto& getGrantedRoles(thread_db* tdbb) const
{
if (testFlag(USR_newrole))
findGrantedRoles(tdbb);
return usr_granted_roles;
}
void makeRoleName(const int dialect)
{
makeRoleName(usr_sql_role_name, dialect);

2
src/jrd/trace/TraceObjects.cpp
View File

@ -234,7 +234,7 @@ void TraceSQLStatementImpl::DSQLParamsImpl::fillParams()
if (m_descs.getCount() || !m_params || m_params->getCount() == 0)
return;
if (!m_stmt->isDml())
if (!m_stmt->getDsqlStatement()->isDml())
{
fb_assert(false);
return;