Frontported improvement #7494 : Firebird performance issue - non necessary index reads.

Also, fixed unregistered bug when scan of multi-segmented descending index with partial match and greater-than condition could miss some records.

src/jrd/btr.cpp

@@ -198,13 +198,13 @@ static ULONG fast_load(thread_db*, IndexCreation&, SelectivityList&);
 
 static index_root_page* fetch_root(thread_db*, WIN*, const jrd_rel*, const RelationPages*);
 static UCHAR* find_node_start_point(btree_page*, temporary_key*, UCHAR*, USHORT*,
-									bool, bool, bool = false, RecordNumber = NO_VALUE);
+									bool, int, bool = false, RecordNumber = NO_VALUE);
 
 static UCHAR* find_area_start_point(btree_page*, const temporary_key*, UCHAR*,
-									USHORT*, bool, bool, RecordNumber = NO_VALUE);
+									USHORT*, bool, int, RecordNumber = NO_VALUE);
 
 static ULONG find_page(btree_page*, const temporary_key*, const index_desc*, RecordNumber = NO_VALUE,
-					   bool = false);
+					   int = 0);
 
 static contents garbage_collect(thread_db*, WIN*, ULONG);
 static void generate_jump_nodes(thread_db*, btree_page*, JumpNodeList*, USHORT,
@@ -717,27 +717,35 @@ static void checkForLowerKeySkip(bool& skipLowerKey,
 	}
 	else
 	{
-		// Check if we have a duplicate node (for the same page)
+		if ((lower.key_length == node.prefix + node.length) ||
-		if (node.prefix < lower.key_length)
+			((lower.key_length <= node.prefix + node.length) && partLower))
+		{
+			const UCHAR* p = node.data, *q = lower.key_data + node.prefix;
+			const UCHAR* const end = lower.key_data + lower.key_length;
+			while (q < end)
+			{
+				if (*p++ != *q++)
 				{
-			if (node.prefix + node.length == lower.key_length)
-				skipLowerKey = (memcmp(node.data, lower.key_data + node.prefix, node.length) == 0);
-			else
 					skipLowerKey = false;
+					break;
 				}
-		else if ((node.prefix == lower.key_length) && node.length)
+			}
+
+			if ((q >= end) && (p < node.data + node.length) && skipLowerKey && partLower)
 			{
-			// In case of multi-segment check segment-number else
+				const bool descending = idx.idx_flags & idx_descending;
-			// it's a different key
+
-			if (partLower)
+				// since key length always is multiplier of (STUFF_COUNT + 1) (for partial
-			{
+				// compound keys) and we passed lower key completely then p pointed
-				const USHORT segnum = idx.idx_count - (UCHAR)((idx.idx_flags & idx_descending) ?
+				// us to the next segment number and we can use this fact to calculate
-					(*node.data) ^ -1 : *node.data);
+				// how many segments is equal to lower key
+				const USHORT segnum = idx.idx_count - (UCHAR) (descending ? ((*p) ^ -1) : *p);
 
 				if (segnum < retrieval->irb_lower_count)
 					skipLowerKey = false;
 			}
-			else
+		}
+		else {
 			skipLowerKey = false;
 		}
 	}
@@ -809,35 +817,7 @@ void BTR_evaluate(thread_db* tdbb, const IndexRetrieval* retrieval, RecordBitmap
 			{
 				IndexNode node;
 				node.readNode(pointer, true);
-
+				checkForLowerKeySkip(skipLowerKey, partLower, node, *lower, idx, retrieval);
-				if ((lower->key_length == node.prefix + node.length) ||
-					((lower->key_length <= node.prefix + node.length) && partLower))
-				{
-					const UCHAR* p = node.data, *q = lower->key_data + node.prefix;
-					const UCHAR* const end = lower->key_data + lower->key_length;
-					while (q < end)
-					{
-						if (*p++ != *q++)
-						{
-							skipLowerKey = false;
-							break;
-						}
-					}
-
-					if ((q >= end) && (p < node.data + node.length) && skipLowerKey && partLower)
-					{
-						// since key length always is multiplier of (STUFF_COUNT + 1) (for partial
-						// compound keys) and we passed lower key completely then p pointed
-						// us to the next segment number and we can use this fact to calculate
-						// how many segments is equal to lower key
-						const USHORT segnum = idx.idx_count - (UCHAR) (descending ? ((*p) ^ -1) : *p);
-
-						if (segnum < retrieval->irb_lower_count)
-							skipLowerKey = false;
-					}
-				}
-				else
-					skipLowerKey = false;
 			}
 		}
 		else
@@ -922,7 +902,7 @@ void BTR_evaluate(thread_db* tdbb, const IndexRetrieval* retrieval, RecordBitmap
 
 
 UCHAR* BTR_find_leaf(btree_page* bucket, temporary_key* key, UCHAR* value,
-					 USHORT* return_value, bool descending, bool retrieval)
+					 USHORT* return_value, bool descending, int retrieval)
 {
 /**************************************
  *
@@ -4313,7 +4293,7 @@ static index_root_page* fetch_root(thread_db* tdbb, WIN* window, const jrd_rel*
 static UCHAR* find_node_start_point(btree_page* bucket, temporary_key* key,
 									UCHAR* value,
 									USHORT* return_value, bool descending,
-									bool retrieval, bool pointer_by_marker,
+									int retrieval, bool pointer_by_marker,
 									RecordNumber find_record_number)
 {
 /**************************************
@@ -4389,9 +4369,21 @@ static UCHAR* find_node_start_point(btree_page* bucket, temporary_key* key,
 			{
 				while (true)
 				{
-					if (q == nodeEnd || (retrieval && p == key_end))
+					if (q == nodeEnd)
 						goto done;
 
+					if (retrieval && p == key_end)
+					{
+						if ((retrieval & irb_partial) && !(retrieval & irb_starting))
+						{
+							// check segment
+							const bool sameSegment = ((p - STUFF_COUNT > key->key_data) && p[-(STUFF_COUNT + 1)] == *q);
+							if (sameSegment)
+								break;
+						}
+						goto done;
+					}
+
 					if (p == key_end || *p > *q)
 						break;
 
@@ -4451,7 +4443,7 @@ done:
 static UCHAR* find_area_start_point(btree_page* bucket, const temporary_key* key,
 									UCHAR* value,
 									USHORT* return_prefix, bool descending,
-									bool retrieval, RecordNumber find_record_number)
+									int retrieval, RecordNumber find_record_number)
 {
 /**************************************
  *
@@ -4557,7 +4549,17 @@ static UCHAR* find_area_start_point(btree_page* bucket, const temporary_key* key
 
 				if (retrieval && keyPointer == keyEnd)
 				{
+					if ((retrieval & irb_partial) && !(retrieval & irb_starting))
+					{
+						// check segment
+						const bool sameSegment = ((keyPointer - STUFF_COUNT > key->key_data) && keyPointer[-(STUFF_COUNT + 1)] == *q);
+						if (!sameSegment)
 							done = true;
+					}
+					else
+					{
+						done = true;
+					}
 					break;
 				}
 
@@ -4664,7 +4666,7 @@ static UCHAR* find_area_start_point(btree_page* bucket, const temporary_key* key
 
 static ULONG find_page(btree_page* bucket, const temporary_key* key,
 					   const index_desc* idx, RecordNumber find_record_number,
-					   bool retrieval)
+					   int retrieval)
 {
 /**************************************
  *
@@ -6486,7 +6488,6 @@ static bool scan(thread_db* tdbb, UCHAR* pointer, RecordBitmap** bitmap, RecordB
 	// stuff the key to the stuff boundary
 	ULONG count;
 	USHORT flag = retrieval->irb_generic;
-	bool partialEquality = false;
 
 	if ((flag & irb_partial) && (flag & irb_equality) &&
 		!(flag & irb_starting) && !(flag & irb_descending))
@@ -6497,7 +6498,6 @@ static bool scan(thread_db* tdbb, UCHAR* pointer, RecordBitmap** bitmap, RecordB
 			key->key_data[key->key_length + i] = 0;
 
 		count += key->key_length;
-		partialEquality = true;
 	}
 	else
 		count = key->key_length;
@@ -6507,13 +6507,13 @@ static bool scan(thread_db* tdbb, UCHAR* pointer, RecordBitmap** bitmap, RecordB
 	count -= key->key_length;
 
 	const bool descending = (flag & irb_descending);
+	const bool equality = (flag & irb_equality);
 	const bool ignoreNulls = (flag & irb_ignore_null_value_key) && (idx->idx_count == 1);
 	bool done = false;
 	bool ignore = false;
 	const bool skipUpperKey = (flag & irb_exclude_upper);
 	const bool partLower = (retrieval->irb_lower_count < idx->idx_count);
 	const bool partUpper = (retrieval->irb_upper_count < idx->idx_count);
-	USHORT upperPrefix = prefix;
 
 	// reset irb_equality flag passed for optimization
 	flag &= ~(irb_equality | irb_ignore_null_value_key);
@@ -6555,7 +6555,7 @@ static bool scan(thread_db* tdbb, UCHAR* pointer, RecordBitmap** bitmap, RecordB
 		else if (node.prefix <= prefix)
 		{
 			prefix = node.prefix;
-			upperPrefix = prefix;
+			USHORT byteInSegment = prefix % (STUFF_COUNT + 1);
 			p = key->key_data + prefix;
 			const UCHAR* q = node.data;
 			USHORT l = node.length;
@@ -6563,50 +6563,53 @@ static bool scan(thread_db* tdbb, UCHAR* pointer, RecordBitmap** bitmap, RecordB
 			{
 				if (skipUpperKey && partUpper)
 				{
-					if (upperPrefix >= key->key_length)
+					if (p >= end_key && byteInSegment == 0)
 					{
 						const USHORT segnum =
 							idx->idx_count - (UCHAR)(descending ? ((*q) ^ -1) : *q) + 1;
 
-						if (segnum >= retrieval->irb_upper_count)
+						if (segnum > retrieval->irb_upper_count)
+							return false;
+
+						if (segnum == retrieval->irb_upper_count && !descending)
 							return false;
 					}
 
-					if (*p == *q)
+					if (++byteInSegment > STUFF_COUNT)
-						upperPrefix++;
+						byteInSegment = 0;
 				}
 
 				if (p >= end_key)
 				{
 					if (flag)
 					{
-						if (partialEquality)
+						// Check if current node bytes is from the same segment as
+						// last byte of the key. If not, we have equality at that
+						// segment. Else, for ascending index, node is greater than
+						// the key and scan should be stopped.
+						// For descending index, the node is less than the key and
+						// scan shoud be continued.
+
+						if ((flag & irb_partial) && !(flag & irb_starting))
 						{
-							// node have no more data, it is equality
+							if ((p - STUFF_COUNT > key->key_data) && (p[-(STUFF_COUNT + 1)] == *q))
-							if (q >= node.data + node.length)
+							{
+								if (descending)
 									break;
 
-							// node contains more bytes than a key, check numbers
-							// of last key segment and current node segment.
-
-							fb_assert(!descending);
-							fb_assert(p - STUFF_COUNT - 1 >= key->key_data);
-
-							const USHORT keySeg = idx->idx_count - p[-STUFF_COUNT - 1];
-							const USHORT nodeSeg = idx->idx_count - *q;
-
-							fb_assert(keySeg <= nodeSeg);
-
-							// If current segment at node is the same as last segment
-							// of the key then node > key.
-							if (keySeg == nodeSeg)
 								return false;
+							}
+
+							if (equality)
+							{
+								const USHORT nodeSeg = idx->idx_count - (UCHAR) (descending ? ((*q) ^ -1) : *q);
 
 								// If node segment belongs to the key segments then key contains
 								// null or empty string and node contains some data.
 								if (nodeSeg < retrieval->irb_upper_count)
 									return false;
 							}
+						}
 						break;
 					}
 					return false;

src/jrd/btr_proto.h

@@ -37,7 +37,7 @@ bool	BTR_description(Jrd::thread_db*, Jrd::jrd_rel*, Ods::index_root_page*, Jrd:
 bool	BTR_check_condition(Jrd::thread_db*, Jrd::index_desc*, Jrd::Record*);
 DSC*	BTR_eval_expression(Jrd::thread_db*, Jrd::index_desc*, Jrd::Record*, bool&);
 void	BTR_evaluate(Jrd::thread_db*, const Jrd::IndexRetrieval*, Jrd::RecordBitmap**, Jrd::RecordBitmap*);
-UCHAR*	BTR_find_leaf(Ods::btree_page*, Jrd::temporary_key*, UCHAR*, USHORT*, bool, bool);
+UCHAR*	BTR_find_leaf(Ods::btree_page*, Jrd::temporary_key*, UCHAR*, USHORT*, bool, int);
 Ods::btree_page*	BTR_find_page(Jrd::thread_db*, const Jrd::IndexRetrieval*, Jrd::win*, Jrd::index_desc*,
 								 Jrd::temporary_key*, Jrd::temporary_key*, bool = true);
 void	BTR_insert(Jrd::thread_db*, Jrd::win*, Jrd::index_insertion*);