Merge pull request #7216 from FirebirdSQL/work/blob_append_v5

New built-in function BLOB_APPEND

src/common/keywords.cpp

@@ -100,6 +100,7 @@ static const TOK tokens[] =
 	{TOK_BIND, "BIND", true},
 	{TOK_BIT_LENGTH, "BIT_LENGTH", false},
 	{TOK_BLOB, "BLOB", false},
+	{TOK_BLOB_APPEND, "BLOB_APPEND", true},
 	{TOK_BLOCK, "BLOCK", true},
 	{TOK_BODY, "BODY", true},
 	{TOK_BOOLEAN, "BOOLEAN", false},

src/dsql/parse.y

@@ -682,6 +682,10 @@ using namespace Firebird;
 %token <metaNamePtr> DEBUG
 %token <metaNamePtr> PKCS_1_5
 
+// tokens added for Firebird 4.0.2
+
+%token <metaNamePtr> BLOB_APPEND
+
 // tokens added for Firebird 5.0
 
 %token <metaNamePtr> TARGET
@@ -8143,6 +8147,7 @@ system_function_std_syntax
 	| BIN_SHL
 	| BIN_SHR
 	| BIN_XOR
+	| BLOB_APPEND
 	| CEIL
 	| CHAR_TO_UUID
 	| COS
@@ -9102,6 +9107,7 @@ non_reserved_word
 	| ZONE
 	| DEBUG				// added in FB 4.0.1
 	| PKCS_1_5
+	| BLOB_APPEND		// added in FB 4.0.2
 	| TARGET			// added in FB 5.0
 	| TIMEZONE_NAME
 	| UNICODE_CHAR

src/jrd/SysFunction.cpp

@@ -223,6 +223,7 @@ bool dscHasData(const dsc* param);
 // specific setParams functions
 void setParamsAsciiVal(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, int argsCount, dsc** args);
 void setParamsBin(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, int argsCount, dsc** args);
+void setParamsBlobAppend(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, int argsCount, dsc** args);
 void setParamsCharToUuid(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, int argsCount, dsc** args);
 void setParamsDateAdd(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, int argsCount, dsc** args);
 void setParamsDateDiff(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, int argsCount, dsc** args);
@@ -258,6 +259,7 @@ void makeAbs(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* r
 void makeAsciiChar(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args);
 void makeBin(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args);
 void makeBinShift(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args);
+void makeBlobAppend(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args);
 void makeCeilFloor(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args);
 void makeDateAdd(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args);
 void makeDateDiff(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args);
@@ -297,6 +299,7 @@ dsc* evlAsciiVal(thread_db* tdbb, const SysFunction* function, const NestValueAr
 dsc* evlAtan2(thread_db* tdbb, const SysFunction* function, const NestValueArray& args, impure_value* impure);
 dsc* evlBin(thread_db* tdbb, const SysFunction* function, const NestValueArray& args, impure_value* impure);
 dsc* evlBinShift(thread_db* tdbb, const SysFunction* function, const NestValueArray& args, impure_value* impure);
+dsc* evlBlobAppend(thread_db* tdbb, const SysFunction* function, const NestValueArray& args, impure_value* impure);
 dsc* evlCeil(thread_db* tdbb, const SysFunction* function, const NestValueArray& args, impure_value* impure);
 dsc* evlCharToUuid(thread_db* tdbb, const SysFunction* function, const NestValueArray& args, impure_value* impure);
 dsc* evlDateAdd(thread_db* tdbb, const SysFunction* function, const NestValueArray& args, impure_value* impure);
@@ -609,6 +612,19 @@ void setParamsAsciiVal(DataTypeUtilBase*, const SysFunction*, int argsCount, dsc
 }
 
 
+void setParamsBlobAppend(DataTypeUtilBase*, const SysFunction*, int argsCount, dsc** args)
+{
+	if (argsCount >= 1 && args[0]->isUnknown())
+		args[0]->makeBlob(isc_blob_text, CS_dynamic);
+
+	for (int i = 1; i < argsCount; ++i)
+	{
+		if (args[i]->isUnknown())
+			args[i]->makeVarying(80, args[0]->getTextType());
+	}
+}
+
+
 void setParamsCharToUuid(DataTypeUtilBase*, const SysFunction*, int argsCount, dsc** args)
 {
 	if (argsCount >= 1 && args[0]->isUnknown())
@@ -1230,6 +1246,18 @@ void makeBinShift(DataTypeUtilBase*, const SysFunction* function, dsc* result,
 }
 
 
+void makeBlobAppend(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, 
+	int argsCount, const dsc** args)
+{
+	USHORT ttype = CS_dynamic;
+
+	if (argsCount > 0 && args[0])
+		ttype = args[0]->getTextType();
+
+	result->makeBlob(isc_blob_text, ttype);
+}
+
+
 void makeCeilFloor(DataTypeUtilBase*, const SysFunction* function, dsc* result,
 	int argsCount, const dsc** args)
 {
@@ -2258,6 +2286,146 @@ HUGEINT getScale(impure_value* impure)
 }
 
 
+static void appendFromBlob(thread_db* tdbb, jrd_tra* transaction, blb* blob, 
+	const dsc* blobDsc, const dsc* srcDsc)
+{
+	if (!srcDsc->dsc_address)
+		return;
+
+	bid* srcBlobID = (bid*)srcDsc->dsc_address;
+	if (srcBlobID->isEmpty())
+		return;
+
+	if (memcmp(blobDsc->dsc_address, srcDsc->dsc_address, sizeof(bid)) == 0)
+		status_exception::raise(Arg::Gds(isc_random) << Arg::Str("Can not append blob to itself"));
+
+	UCharBuffer bpb;
+	BLB_gen_bpb_from_descs(srcDsc, blobDsc, bpb);
+
+	AutoBlb srcBlob(tdbb, blb::open2(tdbb, transaction, srcBlobID, bpb.getCount(), bpb.begin()));
+
+	Database* dbb = tdbb->getDatabase();
+
+	HalfStaticArray<UCHAR, BUFFER_LARGE> buffer;
+	const SLONG buffSize = (srcBlob->getLevel() == 0) ? 
+		MAX(BUFFER_LARGE, srcBlob->blb_length) : dbb->dbb_page_size - BLP_SIZE;
+
+	UCHAR* buff = buffer.getBuffer(buffSize);
+	while (!(srcBlob->blb_flags & BLB_eof))
+	{
+		const SLONG len = srcBlob->BLB_get_data(tdbb, buff, buffSize, false);
+		if (len)
+			blob->BLB_put_data(tdbb, buff, len);
+	}
+}
+
+
+dsc* evlBlobAppend(thread_db* tdbb, const SysFunction* function, const NestValueArray& args, 
+	impure_value* impure)
+{
+	Request* request = tdbb->getRequest();
+	jrd_tra* transaction = request ? request->req_transaction : tdbb->getTransaction();
+	transaction = transaction->getOuter();
+
+	USHORT ttype = tdbb->getCharSet();
+
+	blb* blob = NULL;
+	bid blob_id;
+	dsc blobDsc;
+
+	blob_id.clear();
+	blobDsc.clear();
+
+	const dsc* argDsc = EVL_expr(tdbb, request, args[0]);
+	const bool arg0_null = (request->req_flags & req_null) || (argDsc == NULL);
+
+	if (!arg0_null && argDsc->isBlob())
+		blob_id = *reinterpret_cast<bid*>(argDsc->dsc_address);
+
+	const dsc* declDsc = argDsc;
+	if (!declDsc)
+		declDsc = EVL_assign_to(tdbb, args[0]);
+
+	if (declDsc && declDsc->isBlob())
+	{
+		ttype = declDsc->getTextType();
+		blobDsc.makeBlob(declDsc->getBlobSubType(), ttype, (ISC_QUAD*)&blob_id);
+	}
+	else
+	{
+		if (declDsc && declDsc->isText())
+			ttype = declDsc->getTextType();
+
+		blobDsc.makeBlob(isc_blob_text, ttype, (ISC_QUAD*) &blob_id);
+	}
+
+	bool copyBlob = !blob_id.isEmpty();
+	if (copyBlob)
+	{
+		if (!blob_id.bid_internal.bid_relation_id)
+		{
+			if (!transaction->tra_blobs->locate(blob_id.bid_temp_id()))
+				status_exception::raise(Arg::Gds(isc_bad_segstr_id));
+
+			BlobIndex blobIdx = transaction->tra_blobs->current();
+			if (!blobIdx.bli_materialized && (blobIdx.bli_blob_object->blb_flags & BLB_close_on_read))
+			{
+				blob = blobIdx.bli_blob_object;
+				copyBlob = false;
+			}
+		}
+	}
+
+	if (!blob)
+	{
+		UCharBuffer bpb;
+		BLB_gen_bpb_from_descs(&blobDsc, &blobDsc, bpb);
+		bpb.push(isc_bpb_storage);
+		bpb.push(1);
+		bpb.push(isc_bpb_storage_temp);
+
+		blob = blb::create2(tdbb, transaction, &blob_id, bpb.getCount(), bpb.begin());
+		blob->blb_flags |= BLB_stream | BLB_close_on_read;
+	}
+
+//	if (copyBlob && argDsc && argDsc->isBlob())
+//		appendFromBlob(tdbb, transaction, blob, &blobDsc, argDsc);
+
+	EVL_make_value(tdbb, &blobDsc, impure);
+
+	for (FB_SIZE_T i = 0; i < args.getCount(); i++)
+	{
+		if (i == 0)
+		{
+			if (arg0_null || argDsc->isBlob() && !copyBlob)
+				continue;
+		}
+		else
+		{
+			argDsc = EVL_expr(tdbb, request, args[i]);
+			if ((request->req_flags & req_null) || !argDsc)
+				continue;
+		}
+
+		if (!argDsc->isBlob())
+		{
+			MoveBuffer temp;
+			UCHAR* addr = NULL;
+			SLONG len = MOV_make_string2(tdbb, argDsc, ttype, &addr, temp);
+
+			if (addr)
+				blob->BLB_put_data(tdbb, addr, len);
+		}
+		else
+		{
+			appendFromBlob(tdbb, transaction, blob, &blobDsc, argDsc);
+		}
+	}
+
+	return &impure->vlu_desc;
+}
+
+
 dsc* evlCeil(thread_db* tdbb, const SysFunction*, const NestValueArray& args,
 	impure_value* impure)
 {
@@ -6593,6 +6761,7 @@ const SysFunction SysFunction::functions[] =
 		{"BIN_SHL_ROT", 2, 2, setParamsInteger, makeBinShift, evlBinShift, (void*) funBinShlRot},
 		{"BIN_SHR_ROT", 2, 2, setParamsInteger, makeBinShift, evlBinShift, (void*) funBinShrRot},
 		{"BIN_XOR", 2, -1, setParamsBin, makeBin, evlBin, (void*) funBinXor},
+		{"BLOB_APPEND", 2, -1, setParamsBlobAppend, makeBlobAppend, evlBlobAppend, NULL},
 		{"CEIL", 1, 1, setParamsDblDec, makeCeilFloor, evlCeil, NULL},
 		{"CEILING", 1, 1, setParamsDblDec, makeCeilFloor, evlCeil, NULL},
 		{"CHAR_TO_UUID", 1, 1, setParamsCharToUuid, makeUuid, evlCharToUuid, NULL},

src/jrd/blb.cpp

@@ -59,6 +59,7 @@
 #include "../jrd/dpm_proto.h"
 #include "../jrd/err_proto.h"
 #include "../jrd/evl_proto.h"
+#include "../jrd/exe_proto.h"
 #include "../jrd/filte_proto.h"
 #include "../yvalve/gds_proto.h"
 #include "../jrd/intl_proto.h"
@@ -113,7 +114,12 @@ void blb::BLB_cancel(thread_db* tdbb)
 	// Release filter control resources
 
 	if (blb_flags & BLB_temporary)
+	{
+		if (!(blb_flags & BLB_closed))
+			blb_transaction->tra_temp_blobs_count--;
+
 		delete_blob(tdbb, 0);
+	}
 
 	destroy(true);
 }
@@ -189,11 +195,14 @@ bool blb::BLB_close(thread_db* tdbb)
 
 	SET_TDBB(tdbb);
 
+	const bool alreadyClosed = (blb_flags & BLB_closed);
+
 	// Release filter control resources
 
 	if (blb_filter)
 		BLF_close_blob(tdbb, &blb_filter);
 
+	blb_flags &= ~BLB_close_on_read;
 	blb_flags |= BLB_closed;
 
 	if (!(blb_flags & BLB_temporary))
@@ -202,6 +211,9 @@ bool blb::BLB_close(thread_db* tdbb)
 		return true;
 	}
 
+	if (!alreadyClosed)
+		blb_transaction->tra_temp_blobs_count--;
+
 	if (blb_level == 0)
 	{
 		//Database* dbb = tdbb->getDatabase();
@@ -267,6 +279,40 @@ blb* blb::create2(thread_db* tdbb,
 	Database* dbb = tdbb->getDatabase();
 	CHECK_DBB(dbb);
 
+	const int maxTempBlobs = MAX_TEMP_BLOBS;
+	if (maxTempBlobs > 0 && transaction->tra_temp_blobs_count >= maxTempBlobs)
+	{
+		const Request* request = tdbb->getRequest();
+		string info;
+
+		if (userBlob)
+		{
+			Attachment* att = tdbb->getAttachment();
+			info = "By user application";
+			if (att->att_remote_process.hasData())
+			{
+				info += string(" (") + att->att_remote_process.c_str() + ")";
+			}
+		}
+		else if (request)
+		{
+			const Statement* const statement = request->getStatement();
+			if (statement && statement->sqlText) 
+				info = string("By query: ") + *statement->sqlText;
+
+			string stack;
+			if (EXE_get_stack_trace(request, stack))
+			{
+				info += "\n";
+				info += stack;
+			}
+		}
+
+		gds__log("Too many temporary blobs (%i allowed)\n%s", maxTempBlobs, info.c_str());
+
+		ERR_post(Arg::Gds(isc_random) << Arg::Str("Too many temporary blobs"));
+	}
+
 	// Create a blob large enough to hold a single data page
 	SSHORT from, to;
 	SSHORT from_charset, to_charset;
@@ -322,6 +368,7 @@ blb* blb::create2(thread_db* tdbb,
 
 	blob->blb_space_remaining = blob->blb_clump_size;
 	blob->blb_flags |= BLB_temporary;
+	blob->blb_transaction->tra_temp_blobs_count++;
 
 	if (filter_required)
 	{
@@ -1162,6 +1209,9 @@ void blb::move(thread_db* tdbb, dsc* from_desc, dsc* to_desc,
 
 				if (!blob || !(blob->blb_flags & BLB_closed))
 				{
+					if (blob && (blob->blb_flags & BLB_close_on_read))
+						blob->BLB_close(tdbb);
+					else
 						ERR_post(Arg::Gds(isc_bad_segstr_id));
 				}
 
@@ -1329,7 +1379,7 @@ blb* blb::open2(thread_db* tdbb,
 			 */
 
 			// Search the index of transaction blobs for a match
-			const blb* new_blob = NULL;
+			blb* new_blob = NULL;
 			if (transaction->tra_blobs->locate(blobId.bid_temp_id()))
 			{
 				current = &transaction->tra_blobs->current();
@@ -1344,6 +1394,9 @@ blb* blb::open2(thread_db* tdbb,
 				if (!new_blob || !(new_blob->blb_flags & BLB_temporary) ||
 					!(new_blob->blb_flags & BLB_closed))
 				{
+					if (new_blob && (new_blob->blb_flags & BLB_close_on_read))
+						new_blob->BLB_close(tdbb);
+					else
 						ERR_post(Arg::Gds(isc_bad_segstr_id));
 				}
 
@@ -1544,7 +1597,7 @@ void blb::BLB_put_segment(thread_db* tdbb, const void* seg, USHORT segment_lengt
 
 	// Make sure blob is a temporary blob.  If not, complain bitterly.
 
-	if (!(blb_flags & BLB_temporary))
+	if (!(blb_flags & BLB_temporary) || (blb_flags & BLB_closed))
 		ERR_post(Arg::Gds(isc_cannot_update_old_blob));
 
 	if (blb_filter)

src/jrd/blb.h

@@ -179,7 +179,8 @@ const int BLB_closed		= 8;		// Temporary blob has been closed
 const int BLB_damaged		= 16;		// Blob is busted
 const int BLB_seek			= 32;		// Seek is pending
 const int BLB_large_scan	= 64;		// Blob is larger than page buffer cache
-const int BLB_bulk			= 128;		// Blob created by bulk insert operation
+const int BLB_close_on_read = 128;		// Temporary blob is not closed until read
+const int BLB_bulk			= 256;		// Blob created by bulk insert operation
 
 /* Blob levels are:
 

src/jrd/exe.cpp

@@ -687,6 +687,12 @@ void EXE_receive(thread_db* tdbb,
 							current->bli_request->req_blobs.fastRemove();
 							current->bli_request = NULL;
 						}
+
+						if (!current->bli_materialized &&
+							(current->bli_blob_object->blb_flags & BLB_close_on_read))
+						{
+							current->bli_blob_object->BLB_close(tdbb);
+					}
 					}
 					else
 					{
@@ -1270,10 +1276,9 @@ void EXE_execute_triggers(thread_db* tdbb,
 }
 
 
-static void stuff_stack_trace(const Request* request)
+bool EXE_get_stack_trace(const Request* request, string& sTrace)
 {
-	string sTrace;
+	sTrace = "";
-
 	for (const Request* req = request; req; req = req->req_caller)
 	{
 		const Statement* const statement = req->getStatement();
@@ -1329,7 +1334,14 @@ static void stuff_stack_trace(const Request* request)
 		}
 	}
 
-	if (sTrace.hasData())
+	return sTrace.hasData();
+}
+
+static void stuff_stack_trace(const Request* request)
+{
+	string sTrace;
+
+	if (EXE_get_stack_trace(request, sTrace))
 		ERR_post_nothrow(Arg::Gds(isc_stack_trace) << Arg::Str(sTrace));
 }
 

src/jrd/exe_proto.h

@@ -40,6 +40,8 @@ void EXE_assignment(Jrd::thread_db* tdbb, const Jrd::ValueExprNode* to, dsc* fro
 void EXE_execute_db_triggers(Jrd::thread_db*, Jrd::jrd_tra*, enum TriggerAction);
 void EXE_execute_ddl_triggers(Jrd::thread_db* tdbb, Jrd::jrd_tra* transaction,
 	bool preTriggers, int action);
+bool EXE_get_stack_trace(const Jrd::Request* request, Firebird::string& sTrace);
+
 const Jrd::StmtNode* EXE_looper(Jrd::thread_db* tdbb, Jrd::Request* request,
 	const Jrd::StmtNode* in_node);
 

src/jrd/tra.cpp

@@ -1234,6 +1234,8 @@ void TRA_release_transaction(thread_db* tdbb, jrd_tra* transaction, Jrd::TraceTr
 			blb::release_array(transaction->tra_arrays);
 	}
 
+	fb_assert(transaction->tra_temp_blobs_count == 0);
+
 	if (transaction->tra_pool)
 	{
 		// Iterate the doubly linked list of requests for transaction and null out the transaction references

src/jrd/tra.h

@@ -149,6 +149,7 @@ typedef Firebird::GenericMap<Firebird::Pair<Firebird::NonPooled<SINT64, ULONG> >
 const int DEFAULT_LOCK_TIMEOUT = -1; // infinite
 const char* const TRA_BLOB_SPACE = "fb_blob_";
 const char* const TRA_UNDO_SPACE = "fb_undo_";
+const int MAX_TEMP_BLOBS = 1000;
 
 class jrd_tra : public pool_alloc<type_tra>
 {
@@ -285,6 +286,7 @@ public:
 	UCHAR tra_callback_count;			// callback count for 'execute statement'
 	SSHORT tra_lock_timeout;			// in seconds, -1 means infinite, 0 means NOWAIT
 	ULONG tra_next_blob_id;     		// ID of the previous blob or array created in this transaction
+	ULONG tra_temp_blobs_count;			// Number of active temporary blobs
 	const ISC_TIMESTAMP_TZ tra_timestamp;	// transaction start time
 	Request* tra_requests;				// Doubly linked list of requests active in this transaction
 	MonitoringSnapshot* tra_mon_snapshot;	// Database state snapshot (for monitoring purposes)