firebird/firebird-mirror
firebird/firebird-mirror
8
0
Fork 1
mirror of https://github.com/FirebirdSQL/firebird.git synced 2025-02-02 09:20:39 +01:00
Code Issues

Better processing and optimization if IN <list> predicates (#7707)

Browse Source 
* WIP

* Original (circa 2022) implementation of the IN LIST optimization, with some post-fixes and minor adjustments

* Make it possible to optimize IN <list> for middle segments in compund indices

* Avoid modifying the retrieval structure at runtime, it may be shared among concurrent requests

* Simplify the code a little. Better cost calculation. Support both root-based and sibling-based list scans inside the same plan node.

* Removed the unneeded const casts and other changed as suggested by Adriano
This commit is contained in:
Dmitry Yemanov 2023-09-04 09:13:10 +03:00 committed by GitHub
parent 1973d01f39
commit 0493422c9f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 1002 additions and 234 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

330
src/dsql/BoolNodes.cpp
View File

@ -41,6 +41,7 @@
#include "../dsql/gen_proto.h"
#include "../dsql/make_proto.h"
#include "../dsql/pass1_proto.h"
#include "../dsql/DSqlDataTypeUtil.h"
using namespace Firebird;
using namespace Jrd;
@ -49,8 +50,8 @@ namespace Jrd {
// Maximum members in "IN" list. For eg. SELECT * FROM T WHERE F IN (1, 2, 3, ...)
// Bug 10061, bsriram - 19-Apr-1999
static const int MAX_MEMBER_LIST = 1500;
// Beware: raising the limit beyond the 16-bit boundaries would be an incompatible BLR change.
static const unsigned MAX_MEMBER_LIST = MAX_USHORT;
//--------------------
@ -306,7 +307,20 @@ ComparativeBoolNode::ComparativeBoolNode(MemoryPool& pool, UCHAR aBlrOp,
arg1(aArg1),
arg2(aArg2),
arg3(aArg3),
dsqlSpecialArg(NULL)
dsqlSpecialArg(nullptr)
{
}
ComparativeBoolNode::ComparativeBoolNode(MemoryPool& pool, UCHAR aBlrOp,
ValueExprNode* aArg1, DsqlFlag aDsqlFlag, ExprNode* aSpecialArg)
: TypedNode<BoolExprNode, ExprNode::TYPE_COMPARATIVE_BOOL>(pool),
blrOp(aBlrOp),
dsqlCheckBoolean(false),
dsqlFlag(aDsqlFlag),
arg1(aArg1),
arg2(nullptr),
arg3(nullptr),
dsqlSpecialArg(aSpecialArg)
{
}
@ -355,34 +369,35 @@ BoolExprNode* ComparativeBoolNode::dsqlPass(DsqlCompilerScratch* dsqlScratch)
if (dsqlSpecialArg)
{
ValueListNode* listNode = nodeAs<ValueListNode>(dsqlSpecialArg);
if (listNode)
if (const auto listNode = nodeAs<ValueListNode>(dsqlSpecialArg))
{
int listItemCount = 0;
BoolExprNode* resultNode = NULL;
NestConst<ValueExprNode>* ptr = listNode->items.begin();
for (const NestConst<ValueExprNode>* const end = listNode->items.end();
ptr != end;
++listItemCount, ++ptr)
if (listNode->items.getCount() > MAX_MEMBER_LIST)
{
if (listItemCount >= MAX_MEMBER_LIST)
{
ERRD_post(Arg::Gds(isc_sqlerr) << Arg::Num(-901) <<
Arg::Gds(isc_imp_exc) <<
Arg::Gds(isc_dsql_too_many_values) << Arg::Num(MAX_MEMBER_LIST));
}
ComparativeBoolNode* temp = FB_NEW_POOL(dsqlScratch->getPool()) ComparativeBoolNode(
dsqlScratch->getPool(), blrOp, procArg1, *ptr);
resultNode = PASS1_compose(resultNode, temp, blr_or);
ERRD_post(Arg::Gds(isc_sqlerr) << Arg::Num(-901) <<
Arg::Gds(isc_imp_exc) <<
Arg::Gds(isc_dsql_too_many_values) << Arg::Num(MAX_MEMBER_LIST));
}
if (listNode->items.getCount() == 1)
{
// Convert A IN (B) into A = B
ComparativeBoolNode* const resultNode = FB_NEW_POOL(dsqlScratch->getPool())
ComparativeBoolNode(dsqlScratch->getPool(),
blr_eql, arg1, listNode->items.front());
return resultNode->dsqlPass(dsqlScratch);
}
// Generate the IN LIST boolean
InListBoolNode* const resultNode = FB_NEW_POOL(dsqlScratch->getPool())
InListBoolNode(dsqlScratch->getPool(), procArg1, listNode);
return resultNode->dsqlPass(dsqlScratch);
}
SelectExprNode* selNode = nodeAs<SelectExprNode>(dsqlSpecialArg);
if (selNode)
if (const auto selNode = nodeAs<SelectExprNode>(dsqlSpecialArg))
{
fb_assert(!(selNode->dsqlFlags & RecordSourceNode::DFLAG_SINGLETON));
UCHAR newBlrOp = blr_any;
@ -573,18 +588,13 @@ BoolExprNode* ComparativeBoolNode::pass1(thread_db* tdbb, CompilerScratch* csb)
if ((nodFlags & FLAG_INVARIANT) &&
(!nodeIs<LiteralNode>(arg2) || (arg3 && !nodeIs<LiteralNode>(arg3))))
{
ExprNode* const* ctx_node;
ExprNode* const* end;
for (ctx_node = csb->csb_current_nodes.begin(), end = csb->csb_current_nodes.end();
ctx_node != end; ++ctx_node)
for (const auto& ctxNode : csb->csb_current_nodes)
{
if (nodeAs<RseNode>(*ctx_node))
break;
if (nodeIs<RseNode>(ctxNode))
return this;
}
if (ctx_node >= end)
nodFlags &= ~FLAG_INVARIANT;
nodFlags &= ~FLAG_INVARIANT;
}
}
@ -1143,6 +1153,260 @@ BoolExprNode* ComparativeBoolNode::createRseNode(DsqlCompilerScratch* dsqlScratc
//--------------------
static RegisterBoolNode<InListBoolNode> regInListBoolNode({blr_in_list});
InListBoolNode::InListBoolNode(MemoryPool& pool, ValueExprNode* aArg, ValueListNode* aList)
: TypedNode<BoolExprNode, ExprNode::TYPE_IN_LIST_BOOL>(pool),
arg(aArg),
list(aList)
{
}
DmlNode* InListBoolNode::parse(thread_db* tdbb, MemoryPool& pool, CompilerScratch* csb, const UCHAR blrOp)
{
const auto arg = PAR_parse_value(tdbb, csb);
const auto count = csb->csb_blr_reader.getWord();
const auto list = PAR_args(tdbb, csb, count, count);
return FB_NEW_POOL(pool) InListBoolNode(pool, arg, list);
}
string InListBoolNode::internalPrint(NodePrinter& printer) const
{
BoolExprNode::internalPrint(printer);
NODE_PRINT(printer, blrOp);
NODE_PRINT(printer, arg);
NODE_PRINT(printer, list);
return "InListBoolNode";
}
BoolExprNode* InListBoolNode::dsqlPass(DsqlCompilerScratch* dsqlScratch)
{
const auto procArg = doDsqlPass(dsqlScratch, arg);
const auto procList = doDsqlPass(dsqlScratch, list);
const auto node = FB_NEW_POOL(dsqlScratch->getPool())
InListBoolNode(dsqlScratch->getPool(), procArg, procList);
dsc argDesc;
DsqlDescMaker::fromNode(dsqlScratch, &argDesc, procArg);
dsc listDesc;
DsqlDescMaker::fromList(dsqlScratch, &listDesc, procList, "IN LIST");
if (argDesc.isText() && listDesc.isText())
{
const dsc* descs[] = {&argDesc, &listDesc};
dsc commonDesc;
DSqlDataTypeUtil(dsqlScratch).makeFromList(&commonDesc, "IN LIST",
FB_NELEM(descs), descs);
if (IS_INTL_DATA(&argDesc) || IS_INTL_DATA(&listDesc))
{
const auto charset1 = argDesc.getCharSet();
const auto charset2 = listDesc.getCharSet();
if ((charset1 != CS_BINARY) && (charset2 != CS_BINARY) &&
((charset1 != CS_ASCII) || (charset2 != CS_ASCII)) &&
((charset1 != CS_NONE) || (charset2 != CS_NONE)))
{
const auto ttype = MAX(argDesc.getTextType(), listDesc.getTextType());
commonDesc.setTextType(ttype);
}
}
listDesc = commonDesc;
}
for (auto& item : procList->items)
{
const auto desc = item->getDsqlDesc();
if (!DSC_EQUIV(&listDesc, &desc, true))
{
auto field = FB_NEW_POOL(dsqlScratch->getPool())
dsql_fld(dsqlScratch->getPool());
field->dtype = listDesc.dsc_dtype;
field->scale = listDesc.dsc_scale;
field->subType = listDesc.dsc_sub_type;
field->length = listDesc.dsc_length;
field->flags = (listDesc.dsc_flags & DSC_nullable) ? FLD_nullable : 0;
if (desc.isText() || desc.isBlob())
{
field->textType = listDesc.getTextType();
field->charSetId = listDesc.getCharSet();
field->collationId = listDesc.getCollation();
}
const auto castNode = FB_NEW_POOL(dsqlScratch->getPool())
CastNode(dsqlScratch->getPool(), item, field);
item = castNode->dsqlPass(dsqlScratch);
}
}
// Try to force arg to be same type as list eg: ? = (FIELD, ...) case
for (auto item : procList->items)
PASS1_set_parameter_type(dsqlScratch, node->arg, item, false);
// Try to force list to be same type as arg eg: FIELD = (?, ...) case
for (auto item : procList->items)
PASS1_set_parameter_type(dsqlScratch, item, node->arg, false);
return node;
}
void InListBoolNode::genBlr(DsqlCompilerScratch* dsqlScratch)
{
dsqlScratch->appendUChar(blrOp);
GEN_expr(dsqlScratch, arg);
fb_assert(list->items.getCount() <= MAX_USHORT);
dsqlScratch->appendUShort(list->items.getCount());
for (auto item : list->items)
GEN_expr(dsqlScratch, item);
}
bool InListBoolNode::dsqlMatch(DsqlCompilerScratch* dsqlScratch, const ExprNode* other, bool ignoreMapCast) const
{
if (!BoolExprNode::dsqlMatch(dsqlScratch, other, ignoreMapCast))
return false;
return nodeIs<InListBoolNode>(other);
}
bool InListBoolNode::sameAs(const ExprNode* other, bool ignoreStreams) const
{
const auto otherNode = nodeAs<InListBoolNode>(other);
if (!otherNode)
return false;
return (arg->sameAs(otherNode->arg, ignoreStreams) &&
list->sameAs(otherNode->list, ignoreStreams));
}
BoolExprNode* InListBoolNode::copy(thread_db* tdbb, NodeCopier& copier) const
{
const auto newArg = copier.copy(tdbb, arg);
const auto newList = copier.copy(tdbb, list);
const auto node = FB_NEW_POOL(*tdbb->getDefaultPool())
InListBoolNode(*tdbb->getDefaultPool(), newArg, newList);
node->nodFlags = nodFlags;
return node;
}
BoolExprNode* InListBoolNode::pass1(thread_db* tdbb, CompilerScratch* csb)
{
doPass1(tdbb, csb, arg.getAddress());
nodFlags |= FLAG_INVARIANT;
csb->csb_current_nodes.push(this);
doPass1(tdbb, csb, list.getAddress());
csb->csb_current_nodes.pop();
if (nodFlags & FLAG_INVARIANT)
{
// If there is no top-level RSE present and list items are not constant, unmark node as invariant
// because it may be dependent on data or variables
for (const auto& ctxNode : csb->csb_current_nodes)
{
if (nodeIs<RseNode>(ctxNode))
return this;
}
for (auto item : list->items)
{
while (auto castNode = nodeAs<CastNode>(item))
item = castNode->source;
if (!nodeIs<LiteralNode>(item) && !nodeIs<ParameterNode>(item))
{
nodFlags &= ~FLAG_INVARIANT;
break;
}
}
}
return this;
}
void InListBoolNode::pass2Boolean(thread_db* tdbb, CompilerScratch* csb, std::function<void ()> process)
{
if (nodFlags & FLAG_INVARIANT)
csb->csb_invariants.push(&impureOffset);
process();
if (const auto keyNode = nodeAs<RecordKeyNode>(arg))
{
if (keyNode->aggregate)
ERR_post(Arg::Gds(isc_bad_dbkey));
}
dsc descriptor_a, descriptor_b;
arg->getDesc(tdbb, csb, &descriptor_a);
list->getDesc(tdbb, csb, &descriptor_b);
if (DTYPE_IS_DATE(descriptor_a.dsc_dtype))
arg->nodFlags |= FLAG_DATE;
else if (DTYPE_IS_DATE(descriptor_b.dsc_dtype))
{
for (auto item : list->items)
item->nodFlags |= FLAG_DATE;
}
if (nodFlags & FLAG_INVARIANT)
{
impureOffset = csb->allocImpure<impure_value>();
lookup = FB_NEW_POOL(csb->csb_pool) LookupValueList(csb->csb_pool, list, impureOffset);
}
}
bool InListBoolNode::execute(thread_db* tdbb, Request* request) const
{
if (const auto argDesc = EVL_expr(tdbb, request, arg))
{
if (nodFlags & FLAG_INVARIANT)
{
const auto res = lookup->find(tdbb, request, arg, argDesc);
if (res.isAssigned())
return res.value;
fb_assert(list->items.hasData());
request->req_flags |= req_null;
return false;
}
for (const auto value : list->items)
{
if (const auto valueDesc = EVL_expr(tdbb, request, value))
{
if (!MOV_compare(tdbb, argDesc, valueDesc))
return true;
}
}
}
return false;
}
//--------------------
static RegisterBoolNode<MissingBoolNode> regMissingBoolNode({blr_missing});
MissingBoolNode::MissingBoolNode(MemoryPool& pool, ValueExprNode* aArg, bool aDsqlUnknown)

42
src/dsql/BoolNodes.h
View File

@ -84,8 +84,11 @@ public:
DFLAG_ANSI_ANY
};
ComparativeBoolNode(MemoryPool& pool, UCHAR aBlrOp, ValueExprNode* aArg1 = NULL,
ValueExprNode* aArg2 = NULL, ValueExprNode* aArg3 = NULL);
ComparativeBoolNode(MemoryPool& pool, UCHAR aBlrOp, ValueExprNode* aArg1 = nullptr,
ValueExprNode* aArg2 = nullptr, ValueExprNode* aArg3 = nullptr);
ComparativeBoolNode(MemoryPool& pool, UCHAR aBlrOp, ValueExprNode* aArg1,
DsqlFlag aDsqlFlag, ExprNode* aSpecialArg);
static DmlNode* parse(thread_db* tdbb, MemoryPool& pool, CompilerScratch* csb, const UCHAR blrOp);
@ -138,6 +141,41 @@ public:
};
class InListBoolNode : public TypedNode<BoolExprNode, ExprNode::TYPE_IN_LIST_BOOL>
{
const static UCHAR blrOp = blr_in_list;
public:
InListBoolNode(MemoryPool& pool, ValueExprNode* aArg = nullptr, ValueListNode* aList = nullptr);
static DmlNode* parse(thread_db* tdbb, MemoryPool& pool, CompilerScratch* csb, const UCHAR blrOp);
void getChildren(NodeRefsHolder& holder, bool dsql) const override
{
BoolExprNode::getChildren(holder, dsql);
holder.add(arg);
holder.add(list);
}
Firebird::string internalPrint(NodePrinter& printer) const override;
BoolExprNode* dsqlPass(DsqlCompilerScratch* dsqlScratch) override;
void genBlr(DsqlCompilerScratch* dsqlScratch) override;
BoolExprNode* copy(thread_db* tdbb, NodeCopier& copier) const override;
bool dsqlMatch(DsqlCompilerScratch* dsqlScratch, const ExprNode* other, bool ignoreMapCast) const override;
bool sameAs(const ExprNode* other, bool ignoreStreams) const override;
BoolExprNode* pass1(thread_db* tdbb, CompilerScratch* csb) override;
void pass2Boolean(thread_db* tdbb, CompilerScratch* csb, std::function<void ()> process);
bool execute(thread_db* tdbb, Request* request) const override;
public:
NestConst<ValueExprNode> arg;
NestConst<ValueListNode> list;
NestConst<LookupValueList> lookup;
};
class MissingBoolNode final : public TypedNode<BoolExprNode, ExprNode::TYPE_MISSING_BOOL>
{
public:

8
src/dsql/DdlNodes.epp
View File

@ -4316,9 +4316,7 @@ void CreateDomainNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch
type->resolve(dsqlScratch);
dsqlScratch->domainValue.dsc_dtype = type->dtype;
dsqlScratch->domainValue.dsc_length = type->length;
dsqlScratch->domainValue.dsc_scale = type->scale;
DsqlDescMaker::fromField(&dsqlScratch->domainValue, type);
// run all statements under savepoint control
AutoSavePoint savePoint(tdbb, transaction);
@ -5003,9 +5001,7 @@ void AlterDomainNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScratch,
Arg::Gds(isc_dsql_domain_not_found) << name);
}
dsqlScratch->domainValue.dsc_dtype = localField.dtype;
dsqlScratch->domainValue.dsc_length = localField.length;
dsqlScratch->domainValue.dsc_scale = localField.scale;
DsqlDescMaker::fromField(&dsqlScratch->domainValue, &localField);
FLD.RDB$VALIDATION_SOURCE.NULL = FALSE;
attachment->storeMetaDataBlob(tdbb, transaction, &FLD.RDB$VALIDATION_SOURCE,

83
src/dsql/ExprNodes.cpp
View File

@ -159,9 +159,7 @@ namespace
for (ExprNode** node = csb->csb_current_nodes.end() - 1;
node >= csb->csb_current_nodes.begin(); --node)
{
RseNode* const rseNode = nodeAs<RseNode>(*node);
if (rseNode)
if (const auto rseNode = nodeAs<RseNode>(*node))
{
if (rseNode->containsStream(stream))
break;
@ -194,6 +192,56 @@ static void setParameterInfo(dsql_par* parameter, const dsql_ctx* context);
//--------------------
int SortValueItem::compare(const dsc* desc1, const dsc* desc2)
{
return MOV_compare(JRD_get_thread_data(), desc1, desc2);
}
LookupValueList::LookupValueList(MemoryPool& pool, ValueListNode* values, ULONG impure)
: m_values(pool, values->items.getCount()), m_impureOffset(impure)
{
for (auto& item : values->items)
m_values.add(item);
}
TriState LookupValueList::find(thread_db* tdbb, Request* request, const ValueExprNode* value, const dsc* desc) const
{
const auto impure = request->getImpure<impure_value>(m_impureOffset);
auto sortedList = impure->vlu_misc.vlu_sortedList;
if (!(impure->vlu_flags & VLU_computed))
{
delete impure->vlu_misc.vlu_sortedList;
impure->vlu_misc.vlu_sortedList = nullptr;
sortedList = impure->vlu_misc.vlu_sortedList =
FB_NEW_POOL(*tdbb->getDefaultPool())
SortedValueList(*tdbb->getDefaultPool(), m_values.getCount());
sortedList->setSortMode(FB_ARRAY_SORT_MANUAL);
for (const auto value : m_values)
{
if (const auto valueDesc = EVL_expr(tdbb, request, value))
sortedList->add(SortValueItem(value, valueDesc));
}
sortedList->sort();
impure->vlu_flags |= VLU_computed;
}
if (sortedList->isEmpty())
return TriState();
const auto res = sortedList->exist(SortValueItem(value, desc));
return TriState(res);
}
//--------------------
void Printable::print(NodePrinter& printer) const
{
NodePrinter subPrinter(printer.getIndent() + 1);
@ -449,6 +497,28 @@ Firebird::string ValueListNode::internalPrint(NodePrinter& printer) const
}
void ValueListNode::getDesc(thread_db* tdbb, CompilerScratch* csb, dsc* desc)
{
HalfStaticArray<dsc, INITIAL_CAPACITY> descs;
descs.resize(items.getCount());
unsigned i = 0;
HalfStaticArray<const dsc*, INITIAL_CAPACITY> descPtrs;
descPtrs.resize(items.getCount());
for (auto& value : items)
{
value->getDesc(tdbb, csb, &descs[i]);
descPtrs[i] = &descs[i];
++i;
}
DataTypeUtil(tdbb).makeFromList(desc, "LIST", descPtrs.getCount(), descPtrs.begin());
desc->setNullable(true);
}
//--------------------
@ -4744,9 +4814,7 @@ void DecodeNode::genBlr(DsqlCompilerScratch* dsqlScratch)
dsqlScratch->appendUChar(conditions->items.getCount());
for (auto& condition : conditions->items)
{
condition->genBlr(dsqlScratch);
}
dsqlScratch->appendUChar(values->items.getCount());
@ -11997,11 +12065,12 @@ ValueExprNode* SubstringSimilarNode::pass1(thread_db* tdbb, CompilerScratch* csb
// If there is no top-level RSE present and patterns are not constant, unmark node as invariant
// because it may be dependent on data or variables.
if ((nodFlags & FLAG_INVARIANT) && (!nodeIs<LiteralNode>(pattern) || !nodeIs<LiteralNode>(escape)))
if ((nodFlags & FLAG_INVARIANT) &&
(!nodeIs<LiteralNode>(pattern) || !nodeIs<LiteralNode>(escape)))
{
for (const auto& ctxNode : csb->csb_current_nodes)
{
if (nodeAs<RseNode>(ctxNode))
if (nodeIs<RseNode>(ctxNode))
return this;
}

2
src/dsql/Nodes.h
View File

@ -510,6 +510,7 @@ public:
TYPE_MISSING_BOOL,
TYPE_NOT_BOOL,
TYPE_RSE_BOOL,
TYPE_IN_LIST_BOOL,
// RecordSource types
TYPE_AGGREGATE_SOURCE,
@ -1297,6 +1298,7 @@ public:
}
virtual Firebird::string internalPrint(NodePrinter& printer) const;
virtual void getDesc(thread_db* tdbb, CompilerScratch* csb, dsc* desc);
virtual ValueListNode* dsqlPass(DsqlCompilerScratch* dsqlScratch)
{

18
src/dsql/parse.y
View File

@ -7028,12 +7028,7 @@ comparison_operator
%type <boolExprNode> quantified_predicate
quantified_predicate
: value comparison_operator quantified_flag '(' column_select ')'
{
ComparativeBoolNode* node = newNode<ComparativeBoolNode>($2, $1);
node->dsqlFlag = $3;
node->dsqlSpecialArg = $5;
$$ = node;
}
{ $$ = newNode<ComparativeBoolNode>($2, $1, $3, $5); }
;
%type <cmpBoolFlag> quantified_flag
@ -7124,16 +7119,13 @@ ternary_pattern_predicate
in_predicate
: value IN in_predicate_value
{
ComparativeBoolNode* node = newNode<ComparativeBoolNode>(blr_eql, $1);
node->dsqlFlag = ComparativeBoolNode::DFLAG_ANSI_ANY;
node->dsqlSpecialArg = $3;
$$ = node;
$$ = newNode<ComparativeBoolNode>(blr_eql, $1,
ComparativeBoolNode::DFLAG_ANSI_ANY, $3);
}
| value NOT IN in_predicate_value
{
ComparativeBoolNode* node = newNode<ComparativeBoolNode>(blr_eql, $1);
node->dsqlFlag = ComparativeBoolNode::DFLAG_ANSI_ANY;
node->dsqlSpecialArg = $4;
const auto node = newNode<ComparativeBoolNode>(blr_eql, $1,
ComparativeBoolNode::DFLAG_ANSI_ANY, $4);
$$ = newNode<NotBoolNode>(node);
}
;

3
src/include/firebird/impl/blr.h
View File

@ -168,8 +168,9 @@
#define blr_missing (unsigned char)61
#define blr_unique (unsigned char)62
#define blr_like (unsigned char)63
#define blr_in_list (unsigned char)64
// unused codes: 64..66
// unused codes: 65..66
#define blr_rse (unsigned char)67
#define blr_first (unsigned char)68

15
src/jrd/RecordSourceNodes.cpp
View File

@ -3710,6 +3710,7 @@ static void genDeliverUnmapped(CompilerScratch* csb,
const auto cmpNode = nodeAs<ComparativeBoolNode>(boolean);
const auto missingNode = nodeAs<MissingBoolNode>(boolean);
const auto listNode = nodeAs<InListBoolNode>(boolean);
HalfStaticArray<ValueExprNode*, 2> children;
if (cmpNode &&
@ -3721,6 +3722,8 @@ static void genDeliverUnmapped(CompilerScratch* csb,
children.add(cmpNode->arg1);
children.add(cmpNode->arg2);
}
else if (listNode)
children.add(listNode->arg);
else if (missingNode)
children.add(missingNode->arg);
else
@ -3756,6 +3759,18 @@ static void genDeliverUnmapped(CompilerScratch* csb,
deliverNode = newCmpNode;
}
else if (listNode)
{
const auto newListNode = FB_NEW_POOL(pool) InListBoolNode(pool);
const auto count = listNode->list->items.getCount();
newListNode->list = FB_NEW_POOL(pool) ValueListNode(pool, count);
newChildren.add(newListNode->arg.getAddress());
for (auto item : newListNode->list->items)
newChildren.add(item.getAddress());
deliverNode = newListNode;
}
else if (missingNode)
{
const auto newMissingNode = FB_NEW_POOL(pool) MissingBoolNode(pool);

2
src/jrd/blp.h
View File

@ -89,7 +89,7 @@ static const struct
{"missing", one},
{"unique", one},
{"like", two},
{NULL, NULL},
{"in_list", in_list},
{NULL, NULL},
{NULL, NULL},
{"rse", rse},

245
src/jrd/btr.cpp
View File

@ -45,6 +45,7 @@
#include "../jrd/lck.h"
#include "../jrd/cch.h"
#include "../jrd/sort.h"
#include "../jrd/val.h"
#include "../common/gdsassert.h"
#include "../jrd/btr_proto.h"
#include "../jrd/cch_proto.h"
@ -676,6 +677,90 @@ idx_e IndexKey::compose(Record* record)
}
// IndexScanListIterator class
IndexScanListIterator::IndexScanListIterator(thread_db* tdbb, const IndexRetrieval* retrieval)
: m_tdbb(tdbb), m_retrieval(retrieval),
m_listValues(*tdbb->getDefaultPool(), retrieval->irb_list->getCount()),
m_lowerValues(*tdbb->getDefaultPool()), m_upperValues(*tdbb->getDefaultPool()),
m_iterator(m_listValues.begin())
{
fb_assert(retrieval->irb_list && retrieval->irb_list->getCount());
// Find and store the position of the variable key segment
const auto count = MIN(retrieval->irb_lower_count, retrieval->irb_upper_count);
fb_assert(count);
for (unsigned i = 0; i < count; i++)
{
if (!retrieval->irb_value[i])
{
m_segno = i;
break;
}
}
fb_assert(m_segno < count);
// Copy the list values. Reverse them if index is descending.
m_listValues.assign(retrieval->irb_list->begin(), retrieval->irb_list->getCount());
if (retrieval->irb_generic & irb_descending)
std::reverse(m_listValues.begin(), m_listValues.end());
// Prepare the lower/upper key expressions for evaluation
auto values = m_retrieval->irb_value;
m_lowerValues.assign(values, m_retrieval->irb_lower_count);
fb_assert(!m_lowerValues[m_segno]);
m_lowerValues[m_segno] = *m_iterator;
values += m_retrieval->irb_desc.idx_count;
m_upperValues.assign(values, m_retrieval->irb_upper_count);
fb_assert(!m_upperValues[m_segno]);
m_upperValues[m_segno] = *m_iterator;
}
void IndexScanListIterator::makeKeys(temporary_key* lower, temporary_key* upper)
{
m_lowerValues[m_segno] = *m_iterator;
m_upperValues[m_segno] = *m_iterator;
const auto keyType =
(m_retrieval->irb_desc.idx_flags & idx_unique) ? INTL_KEY_UNIQUE : INTL_KEY_SORT;
// Make the lower bound key
idx_e errorCode = BTR_make_key(m_tdbb, m_retrieval->irb_lower_count, getLowerValues(),
&m_retrieval->irb_desc, lower, keyType);
if (errorCode == idx_e_ok)
{
if (m_retrieval->irb_generic & irb_equality)
{
// If we have an equality search, lower/upper bounds are actually the same key
copy_key(lower, upper);
}
else
{
// Make the upper bound key
errorCode = BTR_make_key(m_tdbb, m_retrieval->irb_upper_count, getUpperValues(),
&m_retrieval->irb_desc, upper, keyType);
}
}
if (errorCode != idx_e_ok)
{
index_desc temp_idx = m_retrieval->irb_desc;
IndexErrorContext context(m_retrieval->irb_relation, &temp_idx);
context.raise(m_tdbb, errorCode);
}
}
void BTR_all(thread_db* tdbb, jrd_rel* relation, IndexDescList& idxList, RelationPages* relPages)
{
/**************************************
@ -1002,7 +1087,6 @@ static void checkForLowerKeySkip(bool& skipLowerKey,
}
}
void BTR_evaluate(thread_db* tdbb, const IndexRetrieval* retrieval, RecordBitmap** bitmap,
RecordBitmap* bitmap_and)
{
@ -1019,26 +1103,30 @@ void BTR_evaluate(thread_db* tdbb, const IndexRetrieval* retrieval, RecordBitmap
**************************************/
SET_TDBB(tdbb);
// Remove ignore_nulls flag for older ODS
//const Database* dbb = tdbb->getDatabase();
index_desc idx;
RelationPages* relPages = retrieval->irb_relation->getPages(tdbb);
WIN window(relPages->rel_pg_space_id, -1);
temporary_key lowerKey, upperKey;
lowerKey.key_flags = 0;
lowerKey.key_length = 0;
upperKey.key_flags = 0;
upperKey.key_length = 0;
AutoPtr<IndexScanListIterator> iterator =
retrieval->irb_list ? FB_NEW_POOL(*tdbb->getDefaultPool())
IndexScanListIterator(tdbb, retrieval) : nullptr;
temporary_key* lower = &lowerKey;
temporary_key* upper = &upperKey;
bool first = true;
BTR_make_bounds(tdbb, retrieval, iterator, lower, upper);
index_desc idx;
btree_page* page = nullptr;
do
{
btree_page* page = BTR_find_page(tdbb, retrieval, &window, &idx, lower, upper, first);
first = false;
if (!page) // scan from the index root
page = BTR_find_page(tdbb, retrieval, &window, &idx, lower, upper);
const bool descending = (idx.idx_flags & idx_descending);
bool skipLowerKey = (retrieval->irb_generic & irb_exclude_lower);
@ -1047,12 +1135,13 @@ void BTR_evaluate(thread_db* tdbb, const IndexRetrieval* retrieval, RecordBitmap
// If there is a starting descriptor, search down index to starting position.
// This may involve sibling buckets if splits are in progress. If there
// isn't a starting descriptor, walk down the left side of the index.
USHORT prefix;
UCHAR* pointer;
if (retrieval->irb_lower_count)
{
while (!(pointer = find_node_start_point(page, lower, 0, &prefix,
idx.idx_flags & idx_descending, (retrieval->irb_generic & (irb_starting | irb_partial)))))
descending, (retrieval->irb_generic & (irb_starting | irb_partial)))))
{
page = (btree_page*) CCH_HANDOFF(tdbb, &window, page->btr_sibling, LCK_read, pag_index);
}
@ -1061,7 +1150,7 @@ void BTR_evaluate(thread_db* tdbb, const IndexRetrieval* retrieval, RecordBitmap
if (retrieval->irb_upper_count)
{
prefix = IndexNode::computePrefix(upper->key_data, upper->key_length,
lower->key_data, lower->key_length);
lower->key_data, lower->key_length);
}
if (skipLowerKey)
@ -1078,9 +1167,9 @@ void BTR_evaluate(thread_db* tdbb, const IndexRetrieval* retrieval, RecordBitmap
skipLowerKey = false;
}
// if there is an upper bound, scan the index pages looking for it
if (retrieval->irb_upper_count)
{
// if there is an upper bound, scan the index pages looking for it
while (scan(tdbb, pointer, bitmap, bitmap_and, &idx, retrieval, prefix, upper,
skipLowerKey, *lower))
{
@ -1147,8 +1236,24 @@ void BTR_evaluate(thread_db* tdbb, const IndexRetrieval* retrieval, RecordBitmap
}
}
// Switch to the new lookup key and continue scanning
// either from the current position or from the root
if (iterator && iterator->getNext(lower, upper))
{
if (!(retrieval->irb_generic & irb_root_list_scan))
continue;
}
else
{
lower = lower->key_next.get();
upper = upper->key_next.get();
}
CCH_RELEASE(tdbb, &window);
} while ((lower = lower->key_next.get()) && (upper = upper->key_next.get()));
page = nullptr;
} while (lower && upper);
}
@ -1176,8 +1281,7 @@ btree_page* BTR_find_page(thread_db* tdbb,
WIN* window,
index_desc* idx,
temporary_key* lower,
temporary_key* upper,
bool makeKeys)
temporary_key* upper)
{
/**************************************
*
@ -1192,51 +1296,6 @@ btree_page* BTR_find_page(thread_db* tdbb,
SET_TDBB(tdbb);
// Generate keys before we get any pages locked to avoid unwind
// problems -- if we already have a key, assume that we
// are looking for an equality
if (retrieval->irb_key)
{
fb_assert(makeKeys);
copy_key(retrieval->irb_key, lower);
copy_key(retrieval->irb_key, upper);
}
else if (makeKeys)
{
idx_e errorCode = idx_e_ok;
const USHORT keyType =
(retrieval->irb_generic & irb_multi_starting) ? INTL_KEY_MULTI_STARTING :
(retrieval->irb_generic & irb_starting) ? INTL_KEY_PARTIAL :
(retrieval->irb_desc.idx_flags & idx_unique) ? INTL_KEY_UNIQUE :
INTL_KEY_SORT;
if (retrieval->irb_upper_count)
{
errorCode = BTR_make_key(tdbb, retrieval->irb_upper_count,
retrieval->irb_value + retrieval->irb_desc.idx_count,
&retrieval->irb_desc, upper,
keyType);
}
if (errorCode == idx_e_ok)
{
if (retrieval->irb_lower_count)
{
errorCode = BTR_make_key(tdbb, retrieval->irb_lower_count,
retrieval->irb_value, &retrieval->irb_desc, lower,
keyType);
}
}
if (errorCode != idx_e_ok)
{
index_desc temp_idx = retrieval->irb_desc; // to avoid constness issues
IndexErrorContext context(retrieval->irb_relation, &temp_idx);
context.raise(tdbb, errorCode, NULL);
}
}
RelationPages* relPages = retrieval->irb_relation->getPages(tdbb);
fb_assert(window->win_page.getPageSpaceID() == relPages->rel_pg_space_id);
@ -1660,6 +1719,68 @@ bool BTR_lookup(thread_db* tdbb, jrd_rel* relation, USHORT id, index_desc* buffe
}
void BTR_make_bounds(thread_db* tdbb, const IndexRetrieval* retrieval,
IndexScanListIterator* iterator,
temporary_key* lower, temporary_key* upper)
{
/**************************************
*
* B T R _ m a k e _ b o u n d s
*
**************************************
*
* Functional description
* Construct search keys for lower/upper bounds for the given retrieval.
*
**************************************/
// If we already have a key, assume that we are looking for an equality
if (retrieval->irb_key)
{
copy_key(retrieval->irb_key, lower);
copy_key(retrieval->irb_key, upper);
}
else
{
idx_e errorCode = idx_e_ok;
const auto idx = &retrieval->irb_desc;
const USHORT keyType =
(retrieval->irb_generic & irb_multi_starting) ? INTL_KEY_MULTI_STARTING :
(retrieval->irb_generic & irb_starting) ? INTL_KEY_PARTIAL :
(retrieval->irb_desc.idx_flags & idx_unique) ? INTL_KEY_UNIQUE :
INTL_KEY_SORT;
if (const auto count = retrieval->irb_upper_count)
{
const auto values = iterator ? iterator->getUpperValues() :
retrieval->irb_value + retrieval->irb_desc.idx_count;
errorCode = BTR_make_key(tdbb, count, values, idx, upper, keyType);
}
if (errorCode == idx_e_ok)
{
if (const auto count = retrieval->irb_lower_count)
{
const auto values = iterator ? iterator->getLowerValues() :
retrieval->irb_value;
errorCode = BTR_make_key(tdbb, count, values, idx, lower, keyType);
}
}
if (errorCode != idx_e_ok)
{
index_desc temp_idx = *idx; // to avoid constness issues
IndexErrorContext context(retrieval->irb_relation, &temp_idx);
context.raise(tdbb, errorCode);
}
}
}
idx_e BTR_make_key(thread_db* tdbb,
USHORT count,
const ValueExprNode* const* exprs,
@ -6546,8 +6667,8 @@ static bool scan(thread_db* tdbb, UCHAR* pointer, RecordBitmap** bitmap, RecordB
const bool partLower = (retrieval->irb_lower_count < idx->idx_count);
const bool partUpper = (retrieval->irb_upper_count < idx->idx_count);
// reset irb_equality flag passed for optimization
flag &= ~(irb_equality | irb_ignore_null_value_key);
// Reset flags this routine does not check in the loop below
flag &= ~(irb_equality | irb_ignore_null_value_key | irb_root_list_scan);
flag &= ~(irb_exclude_lower | irb_exclude_upper);
IndexNode node;
@ -6619,7 +6740,7 @@ static bool scan(thread_db* tdbb, UCHAR* pointer, RecordBitmap** bitmap, RecordB
// segment. Else, for ascending index, node is greater than
// the key and scan should be stopped.
// For descending index, the node is less than the key and
// scan shoud be continued.
// scan should be continued.
if ((flag & irb_partial) && !(flag & irb_starting))
{

60
src/jrd/btr.h
View File

@ -190,7 +190,7 @@ public:
IndexRetrieval(jrd_rel* relation, const index_desc* idx, USHORT count, temporary_key* key)
: irb_relation(relation), irb_index(idx->idx_id),
irb_generic(0), irb_lower_count(count), irb_upper_count(count), irb_key(key),
irb_name(NULL), irb_value(NULL)
irb_name(nullptr), irb_value(nullptr), irb_list(nullptr)
{
memcpy(&irb_desc, idx, sizeof(irb_desc));
}
@ -200,7 +200,8 @@ public:
: irb_relation(relation), irb_index(idx->idx_id),
irb_generic(0), irb_lower_count(0), irb_upper_count(0), irb_key(NULL),
irb_name(FB_NEW_POOL(pool) MetaName(name)),
irb_value(FB_NEW_POOL(pool) ValueExprNode*[idx->idx_count * 2])
irb_value(FB_NEW_POOL(pool) ValueExprNode*[idx->idx_count * 2]),
irb_list(nullptr)
{
memcpy(&irb_desc, idx, sizeof(irb_desc));
}
@ -218,8 +219,9 @@ public:
USHORT irb_lower_count; // Number of segments for retrieval
USHORT irb_upper_count; // Number of segments for retrieval
temporary_key* irb_key; // Key for equality retrieval
MetaName* irb_name; // Index name
ValueExprNode** irb_value;
MetaName* irb_name; // Index name
ValueExprNode** irb_value; // Matching value (for equality search)
LookupValueList* irb_list; // Matching values list (for IN <list>)
};
// Flag values for irb_generic
@ -232,6 +234,7 @@ const int irb_descending = 16; // Base index uses descending order
const int irb_exclude_lower = 32; // exclude lower bound keys while scanning index
const int irb_exclude_upper = 64; // exclude upper bound keys while scanning index
const int irb_multi_starting = 128; // Use INTL_KEY_MULTI_STARTING
const int irb_root_list_scan = 256; // Locate list items from the root
typedef Firebird::HalfStaticArray<float, 4> SelectivityList;
@ -306,8 +309,8 @@ class IndexErrorContext
};
public:
IndexErrorContext(jrd_rel* relation, index_desc* index, const char* indexName = NULL)
: m_relation(relation), m_index(index), m_indexName(indexName), isLocationDefined(false)
IndexErrorContext(jrd_rel* relation, index_desc* index, const char* indexName = nullptr)
: m_relation(relation), m_index(index), m_indexName(indexName)
{}
void setErrorLocation(jrd_rel* relation, USHORT indexId)
@ -317,14 +320,14 @@ public:
m_location.indexId = indexId;
}
void raise(thread_db*, idx_e, Record*);
void raise(thread_db*, idx_e, Record* = nullptr);
private:
jrd_rel* const m_relation;
index_desc* const m_index;
const char* const m_indexName;
Location m_location;
bool isLocationDefined;
bool isLocationDefined = false;
};
// Helper classes to allow efficient evaluation of index conditions/expressions
@ -467,6 +470,47 @@ private:
AutoIndexExpression m_localExpression;
};
// List scan iterator
class IndexScanListIterator
{
public:
IndexScanListIterator(thread_db* tdbb, const IndexRetrieval* retrieval);
bool getNext(temporary_key* lower, temporary_key* upper)
{
if (++m_iterator < m_listValues.end())
{
makeKeys(lower, upper);
return true;
}
m_iterator = nullptr;
return false;
}
ValueExprNode* const* getLowerValues() const
{
return m_lowerValues.begin();
}
ValueExprNode* const* getUpperValues() const
{
return m_upperValues.begin();
}
private:
void makeKeys(temporary_key* lower, temporary_key* upper);
thread_db* const m_tdbb;
const IndexRetrieval* const m_retrieval;
Firebird::HalfStaticArray<ValueExprNode*, 4> m_listValues;
Firebird::HalfStaticArray<ValueExprNode*, 4> m_lowerValues;
Firebird::HalfStaticArray<ValueExprNode*, 4> m_upperValues;
ValueExprNode* const* m_iterator;
USHORT m_segno = MAX_USHORT;
};
} //namespace Jrd
#endif // JRD_BTR_H

6
src/jrd/btr_proto.h
View File

@ -39,13 +39,15 @@ dsc* BTR_eval_expression(Jrd::thread_db*, Jrd::index_desc*, Jrd::Record*);
void BTR_evaluate(Jrd::thread_db*, const Jrd::IndexRetrieval*, Jrd::RecordBitmap**, Jrd::RecordBitmap*);
UCHAR* BTR_find_leaf(Ods::btree_page*, Jrd::temporary_key*, UCHAR*, USHORT*, bool, int);
Ods::btree_page* BTR_find_page(Jrd::thread_db*, const Jrd::IndexRetrieval*, Jrd::win*, Jrd::index_desc*,
Jrd::temporary_key*, Jrd::temporary_key*, bool = true);
Jrd::temporary_key*, Jrd::temporary_key*);
void BTR_insert(Jrd::thread_db*, Jrd::win*, Jrd::index_insertion*);
USHORT BTR_key_length(Jrd::thread_db*, Jrd::jrd_rel*, Jrd::index_desc*);
Ods::btree_page* BTR_left_handoff(Jrd::thread_db*, Jrd::win*, Ods::btree_page*, SSHORT);
bool BTR_lookup(Jrd::thread_db*, Jrd::jrd_rel*, USHORT, Jrd::index_desc*, Jrd::RelationPages*);
void BTR_make_bounds(Jrd::thread_db*, const Jrd::IndexRetrieval*, Jrd::IndexScanListIterator*,
Jrd::temporary_key*, Jrd::temporary_key*);
Jrd::idx_e BTR_make_key(Jrd::thread_db*, USHORT, const Jrd::ValueExprNode* const*, const Jrd::index_desc*,
Jrd::temporary_key*, USHORT);
Jrd::temporary_key*, USHORT);
void BTR_make_null_key(Jrd::thread_db*, const Jrd::index_desc*, Jrd::temporary_key*);
bool BTR_next_index(Jrd::thread_db*, Jrd::jrd_rel*, Jrd::jrd_tra*, Jrd::index_desc*, Jrd::win*);
void BTR_remove(Jrd::thread_db*, Jrd::win*, Jrd::index_insertion*);

12
src/jrd/optimizer/Optimizer.h
View File

@ -378,6 +378,12 @@ public:
break;
}
}
else if (const auto listNode = nodeAs<InListBoolNode>(node))
{
const auto selectivity = REDUCE_SELECTIVITY_FACTOR_EQUALITY *
listNode->list->items.getCount();
return MIN(selectivity, MAXIMUM_SELECTIVITY);
}
else if (nodeIs<MissingBoolNode>(node))
{
return REDUCE_SELECTIVITY_FACTOR_EQUALITY;
@ -517,7 +523,8 @@ enum segmentScanType {
segmentScanEqual,
segmentScanEquivalent,
segmentScanMissing,
segmentScanStarting
segmentScanStarting,
segmentScanList
};
typedef Firebird::HalfStaticArray<BoolExprNode*, OPT_STATIC_ITEMS> MatchedBooleanList;
@ -531,6 +538,7 @@ struct IndexScratchSegment
explicit IndexScratchSegment(MemoryPool& p, const IndexScratchSegment& other)
: lowerValue(other.lowerValue),
upperValue(other.upperValue),
valueList(other.valueList),
excludeLower(other.excludeLower),
excludeUpper(other.excludeUpper),
scope(other.scope),
@ -540,6 +548,7 @@ struct IndexScratchSegment
ValueExprNode* lowerValue = nullptr; // lower bound on index value
ValueExprNode* upperValue = nullptr; // upper bound on index value
LookupValueList* valueList = nullptr; // values to match
bool excludeLower = false; // exclude lower bound value from scan
bool excludeUpper = false; // exclude upper bound value from scan
unsigned scope = 0; // highest scope level
@ -563,6 +572,7 @@ struct IndexScratch
unsigned nonFullMatchedSegments = 0;
bool usePartialKey = false; // Use INTL_KEY_PARTIAL
bool useMultiStartingKeys = false; // Use INTL_KEY_MULTI_STARTING
bool useRootListScan = false;
Firebird::ObjectsArray<IndexScratchSegment> segments;
MatchedBooleanList matches; // matched booleans (partial indices only)

307
src/jrd/optimizer/Retrieval.cpp
View File

@ -124,6 +124,7 @@ IndexScratch::IndexScratch(MemoryPool& p, const IndexScratch& other)
nonFullMatchedSegments(other.nonFullMatchedSegments),
usePartialKey(other.usePartialKey),
useMultiStartingKeys(other.useMultiStartingKeys),
useRootListScan(other.useRootListScan),
segments(p, other.segments),
matches(p, other.matches)
{}
@ -825,6 +826,7 @@ void Retrieval::getInversionCandidates(InversionCandidateList& inversions,
scratch.nonFullMatchedSegments = MAX_INDEX_SEGMENTS + 1;
scratch.usePartialKey = false;
scratch.useMultiStartingKeys = false;
scratch.useRootListScan = false;
const auto idx = scratch.index;
@ -834,6 +836,8 @@ void Retrieval::getInversionCandidates(InversionCandidateList& inversions,
scratch.selectivity = idx->idx_fraction;
bool unique = false;
unsigned listCount = 0;
auto maxSelectivity = scratch.selectivity;
for (unsigned j = 0; j < scratch.segments.getCount(); j++)
{
@ -868,11 +872,23 @@ void Retrieval::getInversionCandidates(InversionCandidateList& inversions,
}
}
if (const auto list = segment.valueList)
{
fb_assert(segment.scanType == segmentScanList);
if (listCount) // we cannot have more than one list matched to an index
break;
listCount = list->getCount();
maxSelectivity = scratch.selectivity;
}
// Check if this is the last usable segment
if (!scratch.usePartialKey &&
(segment.scanType == segmentScanEqual ||
segment.scanType == segmentScanEquivalent ||
segment.scanType == segmentScanMissing))
segment.scanType == segmentScanMissing ||
segment.scanType == segmentScanList))
{
// This is a perfect usable segment thus update root selectivity
scratch.lowerCount++;
@ -914,7 +930,7 @@ void Retrieval::getInversionCandidates(InversionCandidateList& inversions,
{
// This is our last segment that we can use,
// estimate the selectivity
double selectivity = scratch.selectivity;
auto selectivity = scratch.selectivity;
double factor = 1;
switch (segment.scanType)
@ -965,6 +981,7 @@ void Retrieval::getInversionCandidates(InversionCandidateList& inversions,
matches.join(segment.matches);
scratch.nonFullMatchedSegments = idx->idx_count - j;
}
break;
}
}
@ -981,11 +998,33 @@ void Retrieval::getInversionCandidates(InversionCandidateList& inversions,
if (selectivity <= 0)
selectivity = unique ? 1 / cardinality : DEFAULT_SELECTIVITY;
// Calculate the cost (only index pages) for this index
auto cost = DEFAULT_INDEX_COST + selectivity * scratch.cardinality;
if (listCount)
{
// Adjust selectivity based on the list items count
selectivity *= listCount;
selectivity = MIN(selectivity, maxSelectivity);
const auto rootScanCost = DEFAULT_INDEX_COST * listCount +
scratch.cardinality * selectivity;
const auto siblingScanCost = DEFAULT_INDEX_COST +
scratch.cardinality * maxSelectivity * (listCount - 1) / (listCount + 1);
if (rootScanCost < siblingScanCost)
{
cost = rootScanCost;
scratch.useRootListScan = true;
}
else
cost = siblingScanCost;
}
const auto invCandidate = FB_NEW_POOL(getPool()) InversionCandidate(getPool());
invCandidate->unique = unique;
invCandidate->selectivity = selectivity;
// Calculate the cost (only index pages) for this index.
invCandidate->cost = DEFAULT_INDEX_COST + scratch.selectivity * scratch.cardinality;
invCandidate->cost = cost;
invCandidate->nonFullMatchedSegments = scratch.nonFullMatchedSegments;
invCandidate->matchedSegments = MAX(scratch.lowerCount, scratch.upperCount);
invCandidate->indexes = 1;
@ -1062,7 +1101,8 @@ InversionNode* Retrieval::makeIndexScanNode(IndexScratch* indexScratch) const
if (!createIndexScanNodes)
return nullptr;
index_desc* const idx = indexScratch->index;
const auto idx = indexScratch->index;
auto& segments = indexScratch->segments;
// Check whether this is during a compile or during a SET INDEX operation
if (csb)
@ -1097,8 +1137,6 @@ InversionNode* Retrieval::makeIndexScanNode(IndexScratch* indexScratch) const
retrieval->irb_generic |= irb_descending;
}
const auto& segments = indexScratch->segments;
if (const auto count = MAX(indexScratch->lowerCount, indexScratch->upperCount))
{
bool ignoreNullsOnScan = true;
@ -1120,6 +1158,12 @@ InversionNode* Retrieval::makeIndexScanNode(IndexScratch* indexScratch) const
if (segments[i].scanType == segmentScanEquivalent)
ignoreNullsOnScan = false;
if (segments[i].scanType == segmentScanList)
{
fb_assert(!retrieval->irb_list);
retrieval->irb_list = segments[i].valueList;
}
}
}
@ -1130,13 +1174,15 @@ InversionNode* Retrieval::makeIndexScanNode(IndexScratch* indexScratch) const
if (ignoreNullsOnScan && !(idx->idx_runtime_flags & idx_navigate))
retrieval->irb_generic |= irb_ignore_null_value_key;
if (segments[count - 1].scanType == segmentScanStarting)
const auto& lastSegment = segments[count - 1];
if (lastSegment.scanType == segmentScanStarting)
retrieval->irb_generic |= irb_starting;
if (segments[count - 1].excludeLower)
if (lastSegment.excludeLower)
retrieval->irb_generic |= irb_exclude_lower;
if (segments[count - 1].excludeUpper)
if (lastSegment.excludeUpper)
retrieval->irb_generic |= irb_exclude_upper;
}
@ -1149,6 +1195,12 @@ InversionNode* Retrieval::makeIndexScanNode(IndexScratch* indexScratch) const
retrieval->irb_generic |= irb_multi_starting | irb_starting;
}
if (indexScratch->useRootListScan)
{
fb_assert(retrieval->irb_list);
retrieval->irb_generic |= irb_root_list_scan;
}
// Check to see if this is really an equality retrieval
if (retrieval->irb_lower_count == retrieval->irb_upper_count)
{
@ -1546,24 +1598,32 @@ bool Retrieval::matchBoolean(IndexScratch* indexScratch,
const auto cmpNode = nodeAs<ComparativeBoolNode>(boolean);
const auto missingNode = nodeAs<MissingBoolNode>(boolean);
const auto listNode = nodeAs<InListBoolNode>(boolean);
const auto notNode = nodeAs<NotBoolNode>(boolean);
const auto rseNode = nodeAs<RseBoolNode>(boolean);
bool forward = true;
ValueExprNode* value = nullptr;
ValueExprNode* match = nullptr;
ValueListNode* list = nullptr;
if (cmpNode)
{
match = cmpNode->arg1;
value = cmpNode->arg2;
}
else if (listNode)
{
match = listNode->arg;
list = listNode->list;
if (!list->computable(csb, stream, false))
return false;
}
else if (missingNode)
match = missingNode->arg;
else if (notNode || rseNode)
return false;
else
{
fb_assert(false);
fb_assert(notNode || rseNode);
return false;
}
@ -1591,9 +1651,7 @@ bool Retrieval::matchBoolean(IndexScratch* indexScratch,
checkIndexExpression(idx, value) &&
match->computable(csb, stream, false))
{
ValueExprNode* temp = match;
match = value;
value = temp;
std::swap(match, value);
forward = false;
}
else
@ -1611,9 +1669,7 @@ bool Retrieval::matchBoolean(IndexScratch* indexScratch,
fieldNode->fieldStream != stream ||
(value && !value->computable(csb, stream, false)))
{
ValueExprNode* temp = match;
match = value;
value = temp;
std::swap(match, value);
if ((!match || !(fieldNode = nodeAs<FieldNode>(match))) ||
fieldNode->fieldStream != stream ||
@ -1633,10 +1689,14 @@ bool Retrieval::matchBoolean(IndexScratch* indexScratch,
dsc matchDesc, valueDesc;
if (value)
if (value || list)
{
match->getDesc(tdbb, csb, &matchDesc);
value->getDesc(tdbb, csb, &valueDesc);
if (value)
value->getDesc(tdbb, csb, &valueDesc);
else
list->getDesc(tdbb, csb, &valueDesc);
if (!BTR_types_comparable(matchDesc, valueDesc))
return false;
@ -1685,7 +1745,8 @@ bool Retrieval::matchBoolean(IndexScratch* indexScratch,
// AB: If we have already an exact match don't
// override it with worser matches.
if (!((segment->scanType == segmentScanEqual) ||
(segment->scanType == segmentScanEquivalent)))
(segment->scanType == segmentScanEquivalent) ||
(segment->scanType == segmentScanList)))
{
segment->lowerValue = injectCast(csb, value, cast, matchDesc);
segment->upperValue = injectCast(csb, value2, cast2, matchDesc);
@ -1739,7 +1800,8 @@ bool Retrieval::matchBoolean(IndexScratch* indexScratch,
segment->matches.add(boolean);
if (!((segment->scanType == segmentScanEqual) ||
(segment->scanType == segmentScanEquivalent) ||
(segment->scanType == segmentScanBetween)))
(segment->scanType == segmentScanBetween) ||
(segment->scanType == segmentScanList)))
{
if (forward != isDesc) // (forward && !isDesc || !forward && isDesc)
segment->excludeLower = excludeBound;
@ -1770,7 +1832,8 @@ bool Retrieval::matchBoolean(IndexScratch* indexScratch,
segment->matches.add(boolean);
if (!((segment->scanType == segmentScanEqual) ||
(segment->scanType == segmentScanEquivalent) ||
(segment->scanType == segmentScanBetween)))
(segment->scanType == segmentScanBetween) ||
(segment->scanType == segmentScanList)))
{
if (forward != isDesc)
segment->excludeUpper = excludeBound;
@ -1816,6 +1879,27 @@ bool Retrieval::matchBoolean(IndexScratch* indexScratch,
return false;
}
}
else if (listNode)
{
segment->matches.add(boolean);
if (!((segment->scanType == segmentScanEqual) ||
(segment->scanType == segmentScanEquivalent)))
{
if (auto lookup = listNode->lookup)
{
for (auto& item : *lookup)
{
cast = nullptr; // create new cast node for every value
item = injectCast(csb, item, cast, matchDesc);
}
segment->lowerValue = segment->upperValue = nullptr;
segment->valueList = lookup;
segment->scanType = segmentScanList;
segment->excludeLower = false;
segment->excludeUpper = false;
}
}
}
else if (missingNode)
{
segment->matches.add(boolean);
@ -1859,44 +1943,57 @@ InversionCandidate* Retrieval::matchDbKey(BoolExprNode* boolean) const
// If this isn't an equality, it isn't even interesting
const auto cmpNode = nodeAs<ComparativeBoolNode>(boolean);
const auto listNode = nodeAs<InListBoolNode>(boolean);
if (!cmpNode)
return nullptr;
switch (cmpNode->blrOp)
if (cmpNode)
{
case blr_equiv:
case blr_eql:
case blr_gtr:
case blr_geq:
case blr_lss:
case blr_leq:
case blr_between:
break;
switch (cmpNode->blrOp)
{
case blr_equiv:
case blr_eql:
case blr_gtr:
case blr_geq:
case blr_lss:
case blr_leq:
case blr_between:
break;
default:
return nullptr;
default:
return nullptr;
}
}
else if (!listNode)
return nullptr;
// Find the side of the equality that is potentially a dbkey.
// If neither, make the obvious deduction.
SLONG n = 0;
int dbkeyArg = 1;
auto dbkey = findDbKey(cmpNode->arg1, &n);
ValueExprNode* dbkey = nullptr;
if (!dbkey)
if (cmpNode)
{
n = 0;
dbkeyArg = 2;
dbkey = findDbKey(cmpNode->arg2, &n);
dbkey = findDbKey(cmpNode->arg1, &n);
if (!dbkey)
{
n = 0;
dbkeyArg = 2;
dbkey = findDbKey(cmpNode->arg2, &n);
}
if (!dbkey && (cmpNode->blrOp == blr_between))
{
n = 0;
dbkeyArg = 3;
dbkey = findDbKey(cmpNode->arg3, &n);
}
}
if (!dbkey && (cmpNode->blrOp == blr_between))
else
{
n = 0;
dbkeyArg = 3;
dbkey = findDbKey(cmpNode->arg3, &n);
fb_assert(listNode);
dbkey = findDbKey(listNode->arg, &n);
}
if (!dbkey)
@ -1918,55 +2015,63 @@ InversionCandidate* Retrieval::matchDbKey(BoolExprNode* boolean) const
ValueExprNode* lower = nullptr;
ValueExprNode* upper = nullptr;
ValueListNode* list = listNode ? listNode->list : nullptr;
switch (cmpNode->blrOp)
if (cmpNode)
{
case blr_eql:
case blr_equiv:
unique = true;
selectivity = 1 / cardinality;
lower = upper = (dbkeyArg == 1) ? cmpNode->arg2 : cmpNode->arg1;
break;
case blr_gtr:
case blr_geq:
selectivity = REDUCE_SELECTIVITY_FACTOR_GREATER;
if (dbkeyArg == 1)
lower = cmpNode->arg2; // dbkey > arg2
else
upper = cmpNode->arg1; // arg1 < dbkey
break;
case blr_lss:
case blr_leq:
selectivity = REDUCE_SELECTIVITY_FACTOR_LESS;
if (dbkeyArg == 1)
upper = cmpNode->arg2; // dbkey < arg2
else
lower = cmpNode->arg1; // arg1 < dbkey
break;
case blr_between:
if (dbkeyArg == 1) // dbkey between arg2 and arg3
{
selectivity = REDUCE_SELECTIVITY_FACTOR_BETWEEN;
lower = cmpNode->arg2;
upper = cmpNode->arg3;
}
else if (dbkeyArg == 2) // arg1 between dbkey and arg3, or dbkey <= arg1 and arg1 <= arg3
{
selectivity = REDUCE_SELECTIVITY_FACTOR_LESS;
upper = cmpNode->arg1;
}
else if (dbkeyArg == 3) // arg1 between arg2 and dbkey, or arg2 <= arg1 and arg1 <= dbkey
switch (cmpNode->blrOp)
{
case blr_eql:
case blr_equiv:
unique = true;
selectivity = 1 / cardinality;
lower = upper = (dbkeyArg == 1) ? cmpNode->arg2 : cmpNode->arg1;
break;
case blr_gtr:
case blr_geq:
selectivity = REDUCE_SELECTIVITY_FACTOR_GREATER;
lower = cmpNode->arg1;
}
break;
if (dbkeyArg == 1)
lower = cmpNode->arg2; // dbkey > arg2
else
upper = cmpNode->arg1; // arg1 < dbkey
break;
default:
return nullptr;
case blr_lss:
case blr_leq:
selectivity = REDUCE_SELECTIVITY_FACTOR_LESS;
if (dbkeyArg == 1)
upper = cmpNode->arg2; // dbkey < arg2
else
lower = cmpNode->arg1; // arg1 < dbkey
break;
case blr_between:
if (dbkeyArg == 1) // dbkey between arg2 and arg3
{
selectivity = REDUCE_SELECTIVITY_FACTOR_BETWEEN;
lower = cmpNode->arg2;
upper = cmpNode->arg3;
}
else if (dbkeyArg == 2) // arg1 between dbkey and arg3, or dbkey <= arg1 and arg1 <= arg3
{
selectivity = REDUCE_SELECTIVITY_FACTOR_LESS;
upper = cmpNode->arg1;
}
else if (dbkeyArg == 3) // arg1 between arg2 and dbkey, or arg2 <= arg1 and arg1 <= dbkey
{
selectivity = REDUCE_SELECTIVITY_FACTOR_GREATER;
lower = cmpNode->arg1;
}
break;
default:
fb_assert(false);
}
}
else
{
selectivity = list->items.getCount() / cardinality;
}
if (lower && !lower->computable(csb, stream, false))
@ -1975,6 +2080,9 @@ InversionCandidate* Retrieval::matchDbKey(BoolExprNode* boolean) const
if (upper && !upper->computable(csb, stream, false))
return nullptr;
if (list && !list->computable(csb, stream, false))
return nullptr;
const auto invCandidate = FB_NEW_POOL(getPool()) InversionCandidate(getPool());
invCandidate->unique = unique;
invCandidate->selectivity = selectivity;
@ -1985,7 +2093,20 @@ InversionCandidate* Retrieval::matchDbKey(BoolExprNode* boolean) const
if (createIndexScanNodes)
{
if (unique)
if (list)
{
InversionNode* listInversion = nullptr;
for (auto value : list->items)
{
const auto inversion = FB_NEW_POOL(getPool()) InversionNode(value, n);
inversion->impure = csb->allocImpure<impure_inversion>();
listInversion = composeInversion(listInversion, inversion, InversionNode::TYPE_IN);
}
invCandidate->inversion = listInversion;
}
else if (unique)
{
fb_assert(lower == upper);

52
src/jrd/recsrc/IndexTableScan.cpp
View File

@ -116,6 +116,7 @@ void IndexTableScan::close(thread_db* tdbb) const
delete impure->irsb_nav_btr_gc_lock;
impure->irsb_nav_btr_gc_lock = NULL;
}
impure->irsb_nav_page = 0;
if (impure->irsb_nav_lower)
@ -129,6 +130,12 @@ void IndexTableScan::close(thread_db* tdbb) const
delete impure->irsb_nav_upper;
impure->irsb_nav_current_upper = impure->irsb_nav_upper = NULL;
}
if (impure->irsb_iterator)
{
delete impure->irsb_iterator;
impure->irsb_iterator = NULL;
}
}
#ifdef DEBUG_LCK_LIST
// paranoid check
@ -164,6 +171,13 @@ bool IndexTableScan::internalGetRecord(thread_db* tdbb) const
{
impure->irsb_flags &= ~irsb_first;
setPage(tdbb, impure, NULL);
impure->irsb_iterator = m_index->retrieval->irb_list ?
FB_NEW_POOL(*tdbb->getDefaultPool()) IndexScanListIterator(tdbb, m_index->retrieval) :
nullptr;
BTR_make_bounds(tdbb, m_index->retrieval, impure->irsb_iterator,
impure->irsb_nav_lower, impure->irsb_nav_upper);
}
// If this is the first time, start at the beginning
@ -240,6 +254,40 @@ bool IndexTableScan::internalGetRecord(thread_db* tdbb) const
if (retrieval->irb_upper_count &&
compareKeys(idx, key.key_data, key.key_length, &upper, flags) > 0)
{
const auto nextLower = impure->irsb_nav_current_lower;
const auto nextUpper = impure->irsb_nav_current_upper;
if (impure->irsb_iterator && impure->irsb_iterator->getNext(nextLower, nextUpper))
{
if (retrieval->irb_generic & irb_root_list_scan)
{
CCH_RELEASE(tdbb, &window);
page = BTR_find_page(tdbb, retrieval, &window, idx, nextLower, nextUpper);
setPage(tdbb, impure, &window);
}
// If END_BUCKET is reached BTR_find_leaf will return NULL
while (!(nextPointer = BTR_find_leaf(page, nextLower, nullptr, nullptr,
(idx->idx_flags & idx_descending),
(retrieval->irb_generic & (irb_starting | irb_partial)))))
{
page = (Ods::btree_page*) CCH_HANDOFF(tdbb, &window, page->btr_sibling, LCK_read, pag_index);
}
// Update the local keys
key.key_length = nextLower->key_length;
memcpy(key.key_data, nextLower->key_data, key.key_length);
upper.key_length = nextUpper->key_length;
memcpy(upper.key_data, nextUpper->key_data, upper.key_length);
// Update the keys in the impure area
impure->irsb_nav_length = key.key_length;
impure->irsb_nav_upper_length = MIN(m_length + 1, upper.key_length);
memcpy(impure->irsb_nav_data + m_length, upper.key_data, impure->irsb_nav_upper_length);
continue;
}
break;
}
@ -569,7 +617,6 @@ UCHAR* IndexTableScan::openStream(thread_db* tdbb, Impure* impure, win* window)
{
temporary_key* lower = impure->irsb_nav_current_lower;
temporary_key* upper = impure->irsb_nav_current_upper;
const bool firstKeys = lower == impure->irsb_nav_lower;
setPage(tdbb, impure, NULL);
impure->irsb_nav_length = 0;
@ -577,7 +624,8 @@ UCHAR* IndexTableScan::openStream(thread_db* tdbb, Impure* impure, win* window)
// Find the starting leaf page
const IndexRetrieval* const retrieval = m_index->retrieval;
index_desc* const idx = (index_desc*) ((SCHAR*) impure + m_offset);
Ods::btree_page* page = BTR_find_page(tdbb, retrieval, window, idx, lower, upper, firstKeys);
Ods::btree_page* page = BTR_find_page(tdbb, retrieval, window, idx, lower, upper);
setPage(tdbb, impure, window);
// find the upper limit for the search

5
src/jrd/recsrc/RecordSource.cpp
View File

@ -160,7 +160,8 @@ void RecordSource::printInversion(thread_db* tdbb, const InversionNode* inversio
const bool partial = (retrieval->irb_generic & irb_partial);
const bool fullscan = (maxSegs == 0);
const bool unique = uniqueIdx && equality && (minSegs == segCount);
const bool list = (retrieval->irb_list != nullptr);
const bool unique = !list && uniqueIdx && equality && (minSegs == segCount);
string bounds;
if (!unique && !fullscan)
@ -194,7 +195,7 @@ void RecordSource::printInversion(thread_db* tdbb, const InversionNode* inversio
}
plan += "Index " + printName(tdbb, indexName.c_str()) +
(fullscan ? " Full" : unique ? " Unique" : " Range") + " Scan" + bounds;
(fullscan ? " Full" : unique ? " Unique" : list ? " List" : " Range") + " Scan" + bounds;
}
else
{

1
src/jrd/recsrc/RecordSource.h
View File

@ -249,6 +249,7 @@ namespace Jrd
temporary_key* irsb_nav_upper; // upper (possible multiple) key
temporary_key* irsb_nav_current_lower; // current lower key
temporary_key* irsb_nav_current_upper; // current upper key
IndexScanListIterator* irsb_iterator; // key list iterator
USHORT irsb_nav_offset; // page offset of current index node
USHORT irsb_nav_upper_length; // length of upper key value
USHORT irsb_nav_length; // length of expanded key

42
src/jrd/val.h
View File

@ -31,6 +31,7 @@
#include "../include/fb_blk.h"
#include "../common/classes/array.h"
#include "../common/classes/Nullable.h"
#include "../jrd/intl_classes.h"
#include "../jrd/MetaName.h"
#include "../jrd/QualifiedName.h"
@ -56,7 +57,47 @@ namespace Jrd {
class ArrayField;
class blb;
class Request;
class jrd_req;
class jrd_tra;
class thread_db;
class ValueExprNode;
class ValueListNode;
struct SortValueItem
{
SortValueItem(const ValueExprNode* val, const dsc* d)
: value(val), desc(d)
{}
static int compare(const dsc* desc1, const dsc* desc2);
bool operator>(const SortValueItem& other) const
{
return (compare(desc, other.desc) > 0);
}
const ValueExprNode* value;
const dsc* desc;
};
typedef Firebird::SortedArray<SortValueItem> SortedValueList;
class LookupValueList
{
public:
LookupValueList(MemoryPool& pool, ValueListNode* values, ULONG impure);
ULONG getCount() const { return m_values.getCount(); }
ValueExprNode** begin() { return m_values.begin(); }
ValueExprNode** end() { return m_values.end(); }
TriState find(thread_db* tdbb, Request* request,
const ValueExprNode* value, const dsc* desc) const;
private:
Firebird::HalfStaticArray<ValueExprNode*, 4> m_values;
const ULONG m_impureOffset;
};
// Various structures in the impure area
@ -109,6 +150,7 @@ struct impure_value
// Pre-compiled invariant object for pattern matcher functions
Jrd::PatternMatcher* vlu_invariant;
PatternMatcherCache* vlu_patternMatcherCache;
SortedValueList* vlu_sortedList;
} vlu_misc;
void make_long(const SLONG val, const signed char scale = 0);

3
src/yvalve/gds.cpp
View File

@ -412,7 +412,8 @@ static const UCHAR
marks[] = { op_byte, op_literal, op_line, op_verb, 0},
erase[] = { op_erase, 0},
local_table[] = { op_word, op_byte, op_literal, op_byte, op_line, 0},
outer_map[] = { op_outer_map, 0 };
outer_map[] = { op_outer_map, 0 },
in_list[] = { op_verb, op_line, op_word, op_line, op_args, 0};
#include "../jrd/blp.h"