Surface internal optimization modes (all rows vs first rows) at the SQL and configuration levels (#7405)

* Surface internal optimization modes (all rows vs first rows) at the SQL and configuration levels.
* Add session-level control over the optimization strategy
* More informative name as suggested by Adriano
* Cost-based approach for the first-rows optimization mode

builds/install/misc/firebird.conf

@@ -469,6 +469,21 @@
 #InlineSortThreshold = 1000
 
 
+# ----------------------------
+# Defines whether queries should be optimized to retrieve the first records
+# as soon as possible rather than returning the whole dataset as soon as possible.
+# By default retrieval of all rows is implied by the optimizer.
+#
+# Can be overridden at the session level using the SET OPTIMIZE statement
+# or at the SQL statement level by using the OPTIMIZE FOR clause.
+#
+# Per-database configurable.
+#
+# Type: boolean
+#
+#OptimizeForFirstRows = false
+
+
 # ============================
 # Plugin settings
 # ============================

src/common/classes/Nullable.h

@@ -70,11 +70,21 @@ public:
 		return (!specified && !o.specified) || (specified == o.specified && value == o.value);
 	}
 
+	bool operator !=(const BaseNullable<T>& o) const
+	{
+		return !(*this == o);
+	}
+
 	bool operator ==(const T& o) const
 	{
 		return specified && value == o;
 	}
 
+	bool operator !=(const T& o) const
+	{
+		return !(*this == o);
+	}
+
 	void operator =(const T& v)
 	{
 		this->value = v;

src/common/config/config.h

@@ -192,6 +192,7 @@ enum ConfigKey
 	KEY_MAX_STATEMENT_CACHE_SIZE,
 	KEY_PARALLEL_WORKERS,
 	KEY_MAX_PARALLEL_WORKERS,
+	KEY_OPTIMIZE_FOR_FIRST_ROWS,
 	MAX_CONFIG_KEY		// keep it last
 };
 
@@ -310,7 +311,8 @@ constexpr ConfigEntry entries[MAX_CONFIG_KEY] =
 	{TYPE_STRING,	"TempTableDirectory",		false,	""},
 	{TYPE_INTEGER,	"MaxStatementCacheSize",	false,	2 * 1048576},	// bytes
 	{TYPE_INTEGER,	"ParallelWorkers",			true,	1},
-	{TYPE_INTEGER,	"MaxParallelWorkers",		true,	1}
+	{TYPE_INTEGER,	"MaxParallelWorkers",		true,	1},
+	{TYPE_BOOLEAN,	"OptimizeForFirstRows",		false,	false}
 };
 
 
@@ -638,6 +640,8 @@ public:
 	CONFIG_GET_GLOBAL_INT(getParallelWorkers, KEY_PARALLEL_WORKERS);
 
 	CONFIG_GET_GLOBAL_INT(getMaxParallelWorkers, KEY_MAX_PARALLEL_WORKERS);
+
+	CONFIG_GET_PER_DB_BOOL(getOptimizeForFirstRows, KEY_OPTIMIZE_FOR_FIRST_ROWS);
 };
 
 // Implementation of interface to access master configuration file

src/common/keywords.cpp

@@ -339,6 +339,7 @@ static const TOK tokens[] =
 	{TOK_ON, "ON", false},
 	{TOK_ONLY, "ONLY", false},
 	{TOK_OPEN, "OPEN", false},
+	{TOK_OPTIMIZE, "OPTIMIZE", true},
 	{TOK_OPTION, "OPTION", true},
 	{TOK_OR, "OR", false},
 	{TOK_ORDER, "ORDER", false},

src/dsql/BoolNodes.cpp

@@ -1124,7 +1124,7 @@ BoolExprNode* ComparativeBoolNode::createRseNode(DsqlCompilerScratch* dsqlScratc
 	const DsqlContextStack::iterator baseDT(dsqlScratch->derivedContext);
 	const DsqlContextStack::iterator baseUnion(dsqlScratch->unionContext);
 
-	RseNode* rse = PASS1_rse(dsqlScratch, select_expr, false, false);
+	RseNode* rse = PASS1_rse(dsqlScratch, select_expr);
 	rse->flags |= RseNode::FLAG_DSQL_COMPARATIVE;
 
 	// Create a conjunct to be injected.
@@ -1709,7 +1709,7 @@ DmlNode* RseBoolNode::parse(thread_db* tdbb, MemoryPool& pool, CompilerScratch*
 	node->rse->flags |= RseNode::FLAG_SUB_QUERY;
 
 	if (blrOp == blr_any || blrOp == blr_exists) // maybe for blr_unique as well?
-		node->rse->flags |= RseNode::FLAG_OPT_FIRST_ROWS;
+		node->rse->firstRows = true;
 
 	if (csb->csb_currentForNode && csb->csb_currentForNode->parBlrBeginCnt <= 1)
 		node->ownSavepoint = false;
@@ -1744,7 +1744,7 @@ BoolExprNode* RseBoolNode::dsqlPass(DsqlCompilerScratch* dsqlScratch)
 	const DsqlContextStack::iterator base(*dsqlScratch->context);
 
 	RseBoolNode* node = FB_NEW_POOL(dsqlScratch->getPool()) RseBoolNode(dsqlScratch->getPool(), blrOp,
-		PASS1_rse(dsqlScratch, nodeAs<SelectExprNode>(dsqlRse), false, false));
+		PASS1_rse(dsqlScratch, nodeAs<SelectExprNode>(dsqlRse)));
 
 	// Finish off by cleaning up contexts
 	dsqlScratch->context->clear(base);

src/dsql/DdlNodes.epp

@@ -8683,7 +8683,7 @@ void CreateAlterViewNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScra
 
 	dsqlScratch->resetContextStack();
 	++dsqlScratch->contextNumber;
-	RseNode* rse = PASS1_rse(dsqlScratch, selectExpr, false, false);
+	RseNode* rse = PASS1_rse(dsqlScratch, selectExpr);
 
 	dsqlScratch->getBlrData().clear();
 	dsqlScratch->appendUChar(dsqlScratch->isVersion4() ? blr_version4 : blr_version5);

src/dsql/ExprNodes.cpp

@@ -11208,7 +11208,7 @@ ValueExprNode* SubQueryNode::dsqlPass(DsqlCompilerScratch* dsqlScratch)
 
 	const DsqlContextStack::iterator base(*dsqlScratch->context);
 
-	RseNode* rse = PASS1_rse(dsqlScratch, nodeAs<SelectExprNode>(dsqlRse), false, false);
+	RseNode* rse = PASS1_rse(dsqlScratch, nodeAs<SelectExprNode>(dsqlRse));
 
 	SubQueryNode* node = FB_NEW_POOL(dsqlScratch->getPool()) SubQueryNode(dsqlScratch->getPool(), blrOp, rse,
 		rse->dsqlSelectList->items[0], NullNode::instance());

src/dsql/StmtNodes.cpp

@@ -63,6 +63,7 @@
 #include "../dsql/gen_proto.h"
 #include "../dsql/make_proto.h"
 #include "../dsql/pass1_proto.h"
+#include "../dsql/DsqlStatementCache.h"
 
 using namespace Firebird;
 using namespace Jrd;
@@ -1238,7 +1239,7 @@ DeclareCursorNode* DeclareCursorNode::dsqlPass(DsqlCompilerScratch* dsqlScratch)
 	dt->querySpec = dsqlSelect->dsqlExpr;
 	dt->alias = dsqlName.c_str();
 
-	rse = PASS1_derived_table(dsqlScratch, dt, NULL, dsqlSelect->dsqlWithLock, dsqlSelect->dsqlSkipLocked);
+	rse = PASS1_derived_table(dsqlScratch, dt, NULL, dsqlSelect);
 
 	// Assign number and store in the dsqlScratch stack.
 	cursorNumber = dsqlScratch->cursorNumber++;
@@ -4931,7 +4932,7 @@ ForNode* ForNode::dsqlPass(DsqlCompilerScratch* dsqlScratch)
 		dt->querySpec = dsqlSelect->dsqlExpr;
 		dt->alias = dsqlCursor->dsqlName.c_str();
 
-		node->rse = PASS1_derived_table(dsqlScratch, dt, NULL, dsqlSelect->dsqlWithLock, dsqlSelect->dsqlSkipLocked);
+		node->rse = PASS1_derived_table(dsqlScratch, dt, NULL, dsqlSelect);
 
 		dsqlCursor->rse = node->rse;
 		dsqlCursor->cursorNumber = dsqlScratch->cursorNumber++;
@@ -7528,7 +7529,7 @@ StmtNode* StoreNode::internalDsqlPass(DsqlCompilerScratch* dsqlScratch,
 		if (dsqlRse && dsqlScratch->isPsql() && dsqlReturning)
 			selExpr->dsqlFlags |= RecordSourceNode::DFLAG_SINGLETON;
 
-		RseNode* rse = PASS1_rse(dsqlScratch, selExpr, false, false);
+		RseNode* rse = PASS1_rse(dsqlScratch, selExpr);
 		node->dsqlRse = rse;
 		values = rse->dsqlSelectList;
 		needSavePoint = false;
@@ -8206,9 +8207,10 @@ SelectNode* SelectNode::dsqlPass(DsqlCompilerScratch* dsqlScratch)
 {
 	SelectNode* node = FB_NEW_POOL(dsqlScratch->getPool()) SelectNode(dsqlScratch->getPool());
 	node->dsqlForUpdate = dsqlForUpdate;
+	node->dsqlOptimizeForFirstRows = dsqlOptimizeForFirstRows;
 
 	const DsqlContextStack::iterator base(*dsqlScratch->context);
-	node->dsqlRse = PASS1_rse(dsqlScratch, dsqlExpr, dsqlWithLock, dsqlSkipLocked);
+	node->dsqlRse = PASS1_rse(dsqlScratch, dsqlExpr, this);
 	dsqlScratch->context->clear(base);
 
 	if (dsqlForUpdate)
@@ -9170,6 +9172,25 @@ void SetSessionNode::execute(thread_db* tdbb, DsqlRequest* request, jrd_tra** /*
 //--------------------
 
 
+void SetOptimizeNode::execute(thread_db* tdbb, DsqlRequest* /*request*/, jrd_tra** /*traHandle*/) const
+{
+	const auto attachment = tdbb->getAttachment();
+
+	if (attachment->att_opt_first_rows != optimizeMode)
+	{
+		attachment->att_opt_first_rows = optimizeMode;
+
+		// Clear the local compiled statements cache to allow queries
+		// to be re-optimized accordingly to the new rules
+
+		attachment->att_dsql_instance->dbb_statement_cache->purge(tdbb, false);
+	}
+}
+
+
+//--------------------
+
+
 void SetTimeZoneNode::execute(thread_db* tdbb, DsqlRequest* request, jrd_tra** /*traHandle*/) const
 {
 	Attachment* const attachment = tdbb->getAttachment();

src/dsql/StmtNodes.h

@@ -1344,6 +1344,7 @@ public:
 	bool dsqlForUpdate = false;
 	bool dsqlWithLock = false;
 	bool dsqlSkipLocked = false;
+	TriState dsqlOptimizeForFirstRows;
 };
 
 
@@ -1840,6 +1841,37 @@ public:
 };
 
 
+class SetOptimizeNode : public SessionManagementNode
+{
+public:
+	explicit SetOptimizeNode(MemoryPool& pool)
+		: SessionManagementNode(pool)
+	{
+	}
+
+	SetOptimizeNode(MemoryPool& pool, bool mode)
+		: SessionManagementNode(pool),
+		  optimizeMode(mode)
+	{
+	}
+
+public:
+	virtual Firebird::string internalPrint(NodePrinter& printer) const
+	{
+		SessionManagementNode::internalPrint(printer);
+
+		NODE_PRINT(printer, optimizeMode);
+
+		return "SetOptimizeNode";
+	}
+
+	virtual void execute(thread_db* tdbb, DsqlRequest* request, jrd_tra** traHandle) const;
+
+public:
+	TriState optimizeMode;
+};
+
+
 class SetTimeZoneNode : public SessionManagementNode
 {
 public:

src/dsql/gen.cpp

@@ -605,6 +605,12 @@ void GEN_rse(DsqlCompilerScratch* dsqlScratch, RseNode* rse)
 		gen_plan(dsqlScratch, rse->rse_plan);
 	}
 
+	if (rse->firstRows.isAssigned())
+	{
+		dsqlScratch->appendUChar(blr_optimize);
+		dsqlScratch->appendUChar(rse->firstRows.value);
+	}
+
 	dsqlScratch->appendUChar(blr_end);
 }
 

src/dsql/parse-conflicts.txt

@@ -1 +1 @@
-68 shift/reduce conflicts, 19 reduce/reduce conflicts.
+69 shift/reduce conflicts, 21 reduce/reduce conflicts.

src/dsql/parse.y

@@ -689,6 +689,7 @@ using namespace Firebird;
 // tokens added for Firebird 5.0
 
 %token <metaNamePtr> LOCKED
+%token <metaNamePtr> OPTIMIZE
 %token <metaNamePtr> QUARTER
 %token <metaNamePtr> TARGET
 %token <metaNamePtr> TIMEZONE_NAME
@@ -918,6 +919,7 @@ mng_statement
 	| session_reset								{ $$ = $1; }
 	| set_time_zone								{ $$ = $1; }
 	| set_bind									{ $$ = $1; }
+	| set_optimize								{ $$ = $1; }
 	;
 
 
@@ -5481,6 +5483,14 @@ decfloat_trap($setDecFloatTrapsNode)
 		{ $setDecFloatTrapsNode->trap($1); }
 	;
 
+%type <mngNode> set_optimize
+set_optimize
+	: SET OPTIMIZE optimize_mode
+		{ $$ = newNode<SetOptimizeNode>($3); }
+	| SET OPTIMIZE TO DEFAULT
+		{ $$ = newNode<SetOptimizeNode>(); }
+	;
+
 %type <setSessionNode> session_statement
 session_statement
 	: SET SESSION IDLE TIMEOUT long_integer timepart_sesion_idle_tout
@@ -5764,13 +5774,14 @@ ddl_desc
 
 %type <selectNode> select
 select
-	: select_expr for_update_clause lock_clause
+	: select_expr for_update_clause lock_clause optimize_clause
 		{
 			SelectNode* node = newNode<SelectNode>();
 			node->dsqlExpr = $1;
 			node->dsqlForUpdate = $2;
 			node->dsqlWithLock = $3.first;
 			node->dsqlSkipLocked = $3.second;
+			node->dsqlOptimizeForFirstRows = $4;
 			$$ = node;
 		}
 	;
@@ -5799,6 +5810,22 @@ skip_locked_clause_opt
 	| SKIP LOCKED			{ $$ = true; }
 	;
 
+%type <nullableBoolVal>	optimize_clause
+optimize_clause
+	: OPTIMIZE optimize_mode
+		{ $$ = Nullable<bool>::val($2); }
+	| // nothing
+		{ $$ = Nullable<bool>::empty(); }
+	;
+
+%type <boolVal> optimize_mode
+optimize_mode
+	: FOR FIRST ROWS
+		{ $$ = true; }
+	| FOR ALL ROWS
+		{ $$ = false; }
+	;
+
 
 // SELECT expression
 
@@ -9186,6 +9213,7 @@ non_reserved_word
 	| BLOB_APPEND
 	// added in FB 5.0
 	| LOCKED
+	| OPTIMIZE
 	| QUARTER
 	| TARGET
 	| TIMEZONE_NAME

src/dsql/pass1.cpp

@@ -578,15 +578,23 @@ dsql_ctx* PASS1_make_context(DsqlCompilerScratch* dsqlScratch, RecordSourceNode*
 
 // Compile a record selection expression, bumping up the statement scope level everytime an rse is
 // seen. The scope level controls parsing of aliases.
-RseNode* PASS1_rse(DsqlCompilerScratch* dsqlScratch, SelectExprNode* input, bool updateLock, bool skipLocked)
+RseNode* PASS1_rse(DsqlCompilerScratch* dsqlScratch,
+				   SelectExprNode* input,
+				   const SelectNode* select)
 {
 	DEV_BLKCHK(dsqlScratch, dsql_type_req);
 	DEV_BLKCHK(input, dsql_type_nod);
 
+	const bool updateLock = select ? select->dsqlWithLock : false;
+	const bool skipLocked = select ? select->dsqlSkipLocked : false;
+
 	dsqlScratch->scopeLevel++;
 	RseNode* node = pass1_rse(dsqlScratch, input, NULL, NULL, updateLock, skipLocked, 0);
 	dsqlScratch->scopeLevel--;
 
+	if (select)
+		node->firstRows = select->dsqlOptimizeForFirstRows;
+
 	return node;
 }
 
@@ -975,7 +983,7 @@ void PASS1_expand_contexts(DsqlContextStack& contexts, dsql_ctx* context)
 
 // Process derived table which is part of a from clause.
 RseNode* PASS1_derived_table(DsqlCompilerScratch* dsqlScratch, SelectExprNode* input,
-	const char* cte_alias, bool updateLock, bool skipLocked)
+	const char* cte_alias, const SelectNode* select)
 {
 	DEV_BLKCHK(dsqlScratch, dsql_type_req);
 
@@ -1076,7 +1084,7 @@ RseNode* PASS1_derived_table(DsqlCompilerScratch* dsqlScratch, SelectExprNode* i
 			rse = pass1_union(dsqlScratch, unionExpr, NULL, NULL, false, false, 0);
 		}
 		else
-			rse = PASS1_rse(dsqlScratch, input, updateLock, skipLocked);
+			rse = PASS1_rse(dsqlScratch, input, select);
 
 		// Finish off by cleaning up contexts and put them into derivedContext
 		// so create view (ddl) can deal with it.
@@ -1237,7 +1245,7 @@ RseNode* PASS1_derived_table(DsqlCompilerScratch* dsqlScratch, SelectExprNode* i
 			dsqlScratch->currCteAlias ? *dsqlScratch->currCteAlias : NULL;
 		dsqlScratch->resetCTEAlias(alias);
 
-		rse = PASS1_rse(dsqlScratch, input, updateLock, skipLocked);
+		rse = PASS1_rse(dsqlScratch, input, select);
 
 		if (saveCteAlias)
 			dsqlScratch->resetCTEAlias(*saveCteAlias);

src/dsql/pass1_proto.h

@@ -33,6 +33,7 @@ namespace Jrd
 	class RecordSourceNode;
 	class RseNode;
 	class SelectExprNode;
+	class SelectNode;
 	class ValueExprNode;
 	class ValueListNode;
 }
@@ -41,7 +42,8 @@ void PASS1_ambiguity_check(Jrd::DsqlCompilerScratch*, const Jrd::MetaName&, cons
 void PASS1_check_unique_fields_names(Jrd::StrArray& names, const Jrd::CompoundStmtNode* fields);
 Jrd::BoolExprNode* PASS1_compose(Jrd::BoolExprNode*, Jrd::BoolExprNode*, UCHAR);
 Jrd::DeclareCursorNode* PASS1_cursor_name(Jrd::DsqlCompilerScratch*, const Jrd::MetaName&, USHORT, bool);
-Jrd::RseNode* PASS1_derived_table(Jrd::DsqlCompilerScratch*, Jrd::SelectExprNode*, const char*, bool, bool);
+Jrd::RseNode* PASS1_derived_table(Jrd::DsqlCompilerScratch*, Jrd::SelectExprNode*, const char*,
+	const Jrd::SelectNode* = nullptr);
 void PASS1_expand_contexts(Jrd::DsqlContextStack& contexts, Jrd::dsql_ctx* context);
 Jrd::ValueListNode* PASS1_expand_select_list(Jrd::DsqlCompilerScratch*, Jrd::ValueListNode*, Jrd::RecSourceListNode*);
 void PASS1_expand_select_node(Jrd::DsqlCompilerScratch*, Jrd::ExprNode*, Jrd::ValueListNode*, bool);
@@ -55,7 +57,7 @@ bool PASS1_node_match(Jrd::DsqlCompilerScratch*, const Jrd::ExprNode*, const Jrd
 Jrd::DsqlMapNode* PASS1_post_map(Jrd::DsqlCompilerScratch*, Jrd::ValueExprNode*, Jrd::dsql_ctx*,
 	Jrd::WindowClause*);
 Jrd::RecordSourceNode* PASS1_relation(Jrd::DsqlCompilerScratch*, Jrd::RecordSourceNode*);
-Jrd::RseNode* PASS1_rse(Jrd::DsqlCompilerScratch*, Jrd::SelectExprNode*, bool, bool);
+Jrd::RseNode* PASS1_rse(Jrd::DsqlCompilerScratch*, Jrd::SelectExprNode*, const Jrd::SelectNode* = nullptr);
 bool PASS1_set_parameter_type(Jrd::DsqlCompilerScratch*, Jrd::ValueExprNode*, std::function<void (dsc*)>, bool);
 bool PASS1_set_parameter_type(Jrd::DsqlCompilerScratch*, Jrd::ValueExprNode*, NestConst<Jrd::ValueExprNode>, bool);
 Jrd::ValueListNode* PASS1_sort(Jrd::DsqlCompilerScratch*, Jrd::ValueListNode*, Jrd::ValueListNode*);

src/include/firebird/impl/blr.h

@@ -219,8 +219,9 @@
 #define blr_ansi_like		(unsigned char)108
 #define blr_scrollable		(unsigned char) 109
 #define blr_lateral_rse		(unsigned char) 110
+#define blr_optimize		(unsigned char) 111
 
-// unused codes: 111..117
+// unused codes: 112..117
 
 ///#define blr_run_count		(unsigned char)118
 #define blr_rs_stream		(unsigned char)119

src/jrd/Attachment.h

@@ -639,6 +639,7 @@ public:
 	USHORT att_original_timezone;
 	USHORT att_current_timezone;
 	int att_parallel_workers;
+	TriState att_opt_first_rows;
 
 	PageToBufferMap* att_bdb_cache;			// managed in CCH, created in att_pool, freed with it
 

src/jrd/RecordSourceNodes.cpp

@@ -1645,7 +1645,7 @@ RecordSource* AggregateSourceNode::compile(thread_db* tdbb, Optimizer* opt, bool
 		// 10-Aug-2004. Nickolay Samofatov - Unneeded nulls seem to be skipped somehow.
 		aggregate->nullOrder.add(NULLS_DEFAULT);
 
-		rse->flags |= RseNode::FLAG_OPT_FIRST_ROWS;
+		rse->firstRows = true;
 	}
 
 	RecordSource* const nextRsb = opt->compile(rse, &deliverStack);
@@ -3439,7 +3439,7 @@ string SelectExprNode::internalPrint(NodePrinter& printer) const
 RseNode* SelectExprNode::dsqlPass(DsqlCompilerScratch* dsqlScratch)
 {
 	fb_assert(dsqlFlags & DFLAG_DERIVED);
-	return PASS1_derived_table(dsqlScratch, this, NULL, false, false);
+	return PASS1_derived_table(dsqlScratch, this, NULL);
 }
 
 
@@ -3508,7 +3508,7 @@ static RecordSourceNode* dsqlPassRelProc(DsqlCompilerScratch* dsqlScratch, Recor
 	dsqlScratch->currCtes.push(cte);
 
 	RseNode* derivedNode = PASS1_derived_table(dsqlScratch,
-		cte, (isRecursive ? relAlias.c_str() : NULL), false, false);
+		cte, (isRecursive ? relAlias.c_str() : NULL));
 
 	if (!isRecursive)
 		cte->alias = saveCteName;

src/jrd/RecordSourceNodes.h

@@ -724,10 +724,9 @@ public:
 		FLAG_WRITELOCK			= 0x04,	// locked for write
 		FLAG_SCROLLABLE			= 0x08,	// scrollable cursor
 		FLAG_DSQL_COMPARATIVE	= 0x10,	// transformed from DSQL ComparativeBoolNode
-		FLAG_OPT_FIRST_ROWS		= 0x20,	// optimize retrieval for first rows
-		FLAG_LATERAL			= 0x40,	// lateral derived table
-		FLAG_SKIP_LOCKED		= 0x80,	// skip locked
-		FLAG_SUB_QUERY			= 0x100	// sub-query
+		FLAG_LATERAL			= 0x20,	// lateral derived table
+		FLAG_SKIP_LOCKED		= 0x40,	// skip locked
+		FLAG_SUB_QUERY			= 0x80	// sub-query
 	};
 
 	bool isInvariant() const
@@ -767,25 +766,8 @@ public:
 
 	explicit RseNode(MemoryPool& pool)
 		: TypedNode<RecordSourceNode, RecordSourceNode::TYPE_RSE>(pool),
-		  dsqlFirst(NULL),
-		  dsqlSkip(NULL),
-		  dsqlDistinct(NULL),
-		  dsqlSelectList(NULL),
-		  dsqlFrom(NULL),
-		  dsqlWhere(NULL),
-		  dsqlJoinUsing(NULL),
-		  dsqlGroup(NULL),
-		  dsqlHaving(NULL),
-		  dsqlNamedWindows(NULL),
-		  dsqlOrder(NULL),
-		  dsqlStreams(NULL),
-		  rse_invariants(NULL),
-		  rse_relations(pool),
-		  flags(0),
-		  rse_jointype(blr_inner),
-		  dsqlExplicitJoin(false)
-	{
-	}
+		  rse_relations(pool)
+	{}
 
 	RseNode* clone(MemoryPool& pool)
 	{
@@ -817,6 +799,7 @@ public:
 		obj->rse_invariants = rse_invariants;
 		obj->flags = flags;
 		obj->rse_relations = rse_relations;
+		obj->firstRows = firstRows;
 
 		return obj;
 	}
@@ -886,9 +869,10 @@ public:
 	NestConst<ValueListNode> dsqlJoinUsing;
 	NestConst<ValueListNode> dsqlGroup;
 	NestConst<BoolExprNode> dsqlHaving;
-	NamedWindowsClause* dsqlNamedWindows;
 	NestConst<ValueListNode> dsqlOrder;
 	NestConst<RecSourceListNode> dsqlStreams;
+	NamedWindowsClause* dsqlNamedWindows = nullptr;
+	bool dsqlExplicitJoin = false;
 	NestConst<ValueExprNode> rse_first;
 	NestConst<ValueExprNode> rse_skip;
 	NestConst<BoolExprNode> rse_boolean;
@@ -898,9 +882,9 @@ public:
 	NestConst<PlanNode> rse_plan;		// user-specified access plan
 	NestConst<VarInvariantArray> rse_invariants; // Invariant nodes bound to top-level RSE
 	Firebird::Array<NestConst<RecordSourceNode> > rse_relations;
-	USHORT flags;
-	USHORT rse_jointype;		// inner, left, full
-	bool dsqlExplicitJoin;
+	USHORT flags = 0;
+	USHORT rse_jointype = blr_inner;	// inner, left, full
+	TriState firstRows;					// optimize for first rows
 };
 
 class SelectExprNode final : public TypedNode<RecordSourceNode, RecordSourceNode::TYPE_SELECT_EXPR>

src/jrd/optimizer/InnerJoin.cpp

@@ -192,8 +192,21 @@ void InnerJoin::estimateCost(unsigned position,
 	const auto tail = &csb->csb_rpt[stream->number];
 	const auto streamCardinality = tail->csb_cardinality;
 
+	auto currentCardinality = streamCardinality * candidate->selectivity;
+	auto currentCost = candidate->cost;
+
+	// Unless an external sort is to be applied, adjust estimated cost and cardinality
+	// accordingly to the "first-rows" retrieval (if specified)
+	if ((!sort || candidate->navigated) && optimizer->favorFirstRows())
+	{
+		currentCost -= DEFAULT_INDEX_COST;
+		currentCost /= MAX(currentCardinality, MINIMUM_CARDINALITY);
+		currentCost += DEFAULT_INDEX_COST;
+		currentCardinality = MINIMUM_CARDINALITY;
+	}
+
 	// Calculate the nested loop cost, it's our default option
-	const auto loopCost = candidate->cost * cardinality;
+	const auto loopCost = currentCost * cardinality;
 	cost = loopCost;
 
 	if (position)
@@ -208,8 +221,7 @@ void InnerJoin::estimateCost(unsigned position,
 			// hashing cost
 			hashCardinality * (COST_FACTOR_MEMCOPY + COST_FACTOR_HASHING) +
 			// probing + copying cost
-			cardinality * (COST_FACTOR_HASHING +
-				candidate->selectivity * streamCardinality * COST_FACTOR_MEMCOPY);
+			cardinality * (COST_FACTOR_HASHING + currentCardinality * COST_FACTOR_MEMCOPY);
 
 		if (hashCost <= loopCost && hashCardinality <= HashJoin::maxCapacity())
 		{
@@ -242,8 +254,7 @@ void InnerJoin::estimateCost(unsigned position,
 		}
 	}
 
-	const auto resultingCardinality = streamCardinality * candidate->selectivity;
-	cardinality = MAX(resultingCardinality, MINIMUM_CARDINALITY);
+	cardinality = MAX(currentCardinality, MINIMUM_CARDINALITY);
 }
 
 
@@ -264,24 +275,12 @@ bool InnerJoin::findJoinOrder()
 	printStartOrder();
 #endif
 
-	int filters = 0, navigations = 0;
-
 	for (const auto innerStream : innerStreams)
 	{
 		if (!innerStream->used)
 		{
 			remainingStreams++;
 
-			const int currentFilter = innerStream->isFiltered() ? 1 : 0;
-
-			if (navigations && currentFilter)
-				navigations = 0;
-
-			filters += currentFilter;
-
-			if (innerStream->baseNavigated && currentFilter == filters)
-				navigations++;
-
 			if (innerStream->isIndependent())
 			{
 				if (!bestCount || innerStream->baseCost < bestCost)
@@ -303,24 +302,11 @@ bool InnerJoin::findJoinOrder()
 		{
 			if (!innerStream->used)
 			{
-				// If optimization for first rows has been requested and index navigations are
-				// possible, then consider only join orders starting with a navigational stream.
-				// Except cases when other streams have local predicates applied.
+				indexedRelationships.clear();
+				findBestOrder(0, innerStream, indexedRelationships, 0.0, 1.0);
 
-				const int currentFilter = innerStream->isFiltered() ? 1 : 0;
-
-				if (!optimizer->favorFirstRows() || !navigations ||
-					(innerStream->baseNavigated && currentFilter == filters))
-				{
-					indexedRelationships.clear();
-					findBestOrder(0, innerStream, indexedRelationships, 0.0, 1.0);
-
-					if (plan)
-					{
-						// If a explicit PLAN was specified we should be ready;
-						break;
-					}
-				}
+				if (plan) // if an explicit PLAN was specified we should be ready
+					break;
 			}
 		}
 	}

src/jrd/optimizer/Optimizer.cpp

@@ -562,9 +562,10 @@ namespace
 // Constructor
 //
 
-Optimizer::Optimizer(thread_db* aTdbb, CompilerScratch* aCsb, RseNode* aRse)
+Optimizer::Optimizer(thread_db* aTdbb, CompilerScratch* aCsb, RseNode* aRse, bool parentFirstRows)
 	: PermanentStorage(*aTdbb->getDefaultPool()),
 	  tdbb(aTdbb), csb(aCsb), rse(aRse),
+	  firstRows(rse->firstRows.orElse(parentFirstRows)),
 	  compileStreams(getPool()),
 	  bedStreams(getPool()),
 	  keyStreams(getPool()),
@@ -572,6 +573,24 @@ Optimizer::Optimizer(thread_db* aTdbb, CompilerScratch* aCsb, RseNode* aRse)
 	  outerStreams(getPool()),
 	  conjuncts(getPool())
 {
+    // Ignore optimization for first rows in impossible cases
+	if (firstRows)
+	{
+		// Projection is currently always performed using an external sort,
+		// so all underlying records will be fetched anyway
+		if (rse->rse_projection)
+			firstRows = false;
+		// Aggregation without GROUP BY will also cause all records to be fetched.
+		// Exception is when MIN/MAX functions could be mapped to an index,
+		// but this is handled separately inside AggregateSourceNode::compile().
+		else if (rse->rse_relations.getCount() == 1)
+		{
+			const auto subRse = rse->rse_relations[0];
+			const auto aggregate = nodeAs<AggregateSourceNode>(subRse);
+			if (aggregate && !aggregate->group)
+				firstRows = false;
+		}
+	}
 }
 
 
@@ -599,7 +618,7 @@ Optimizer::~Optimizer()
 
 RecordSource* Optimizer::compile(RseNode* subRse, BoolExprNodeStack* parentStack)
 {
-	Optimizer subOpt(tdbb, csb, subRse);
+	Optimizer subOpt(tdbb, csb, subRse, firstRows);
 	const auto rsb = subOpt.compile(parentStack);
 
 	if (parentStack && subOpt.isInnerJoin())

src/jrd/optimizer/Optimizer.h

@@ -409,7 +409,19 @@ public:
 
 	static RecordSource* compile(thread_db* tdbb, CompilerScratch* csb, RseNode* rse)
 	{
-		return Optimizer(tdbb, csb, rse).compile(nullptr);
+		bool firstRows = false;
+
+		// System requests should not be affected by user-specified settings
+		if (!(csb->csb_g_flags & csb_internal))
+		{
+			const auto dbb = tdbb->getDatabase();
+			const auto defaultFirstRows = dbb->dbb_config->getOptimizeForFirstRows();
+
+			const auto attachment = tdbb->getAttachment();
+			firstRows = attachment->att_opt_first_rows.orElse(defaultFirstRows);
+		}
+
+		return Optimizer(tdbb, csb, rse, firstRows).compile(nullptr);
 	}
 
 	~Optimizer();
@@ -455,7 +467,7 @@ public:
 
 	bool favorFirstRows() const
 	{
-		return (rse->flags & RseNode::FLAG_OPT_FIRST_ROWS) != 0;
+		return firstRows;
 	}
 
 	RecordSource* applyLocalBoolean(RecordSource* rsb,
@@ -471,7 +483,7 @@ public:
 	void printf(const char* format, ...);
 
 private:
-	Optimizer(thread_db* aTdbb, CompilerScratch* aCsb, RseNode* aRse);
+	Optimizer(thread_db* aTdbb, CompilerScratch* aCsb, RseNode* aRse, bool parentFirstRows);
 
 	RecordSource* compile(BoolExprNodeStack* parentStack);
 
@@ -505,6 +517,8 @@ private:
 	CompilerScratch* const csb;
 	RseNode* const rse;
 
+	bool firstRows = false;					// optimize for first rows
+
 	FILE* debugFile = nullptr;
 	unsigned baseConjuncts = 0;				// number of conjuncts in our rse, next conjuncts are distributed parent
 	unsigned baseParentConjuncts = 0;		// number of conjuncts in our rse + distributed with parent, next are parent

src/jrd/par.cpp

@@ -1346,7 +1346,7 @@ RseNode* PAR_rse(thread_db* tdbb, CompilerScratch* csb, SSHORT rse_op)
 			if (rse_op == blr_rs_stream)
 				PAR_syntax_error(csb, "RecordSelExpr stream clause");
 			rse->rse_first = PAR_parse_value(tdbb, csb);
-			rse->flags |= RseNode::FLAG_OPT_FIRST_ROWS;
+			rse->firstRows = true;
 			break;
 
 		case blr_skip:
@@ -1418,6 +1418,10 @@ RseNode* PAR_rse(thread_db* tdbb, CompilerScratch* csb, SSHORT rse_op)
 			rse->flags |= RseNode::FLAG_SKIP_LOCKED;
 			break;
 
+		case blr_optimize:
+			rse->firstRows = (csb->csb_blr_reader.getByte() != 0);
+			break;
+
 		default:
 			if (op == (UCHAR) blr_end)
 			{