diff --git a/src/common/keywords.cpp b/src/common/keywords.cpp index 5eb003d4a0..f27a9f3010 100644 --- a/src/common/keywords.cpp +++ b/src/common/keywords.cpp @@ -100,6 +100,7 @@ static const TOK tokens[] = {TOK_BIND, "BIND", true}, {TOK_BIT_LENGTH, "BIT_LENGTH", false}, {TOK_BLOB, "BLOB", false}, + {TOK_BLOB_APPEND, "BLOB_APPEND", true}, {TOK_BLOCK, "BLOCK", true}, {TOK_BODY, "BODY", true}, {TOK_BOOLEAN, "BOOLEAN", false}, diff --git a/src/dsql/parse.y b/src/dsql/parse.y index 55e590a41f..7059de994a 100644 --- a/src/dsql/parse.y +++ b/src/dsql/parse.y @@ -682,6 +682,10 @@ using namespace Firebird; %token DEBUG %token PKCS_1_5 +// tokens added for Firebird 4.0.2 + +%token BLOB_APPEND + // tokens added for Firebird 5.0 %token TARGET @@ -8143,6 +8147,7 @@ system_function_std_syntax | BIN_SHL | BIN_SHR | BIN_XOR + | BLOB_APPEND | CEIL | CHAR_TO_UUID | COS @@ -9102,6 +9107,7 @@ non_reserved_word | ZONE | DEBUG // added in FB 4.0.1 | PKCS_1_5 + | BLOB_APPEND // added in FB 4.0.2 | TARGET // added in FB 5.0 | TIMEZONE_NAME | UNICODE_CHAR diff --git a/src/jrd/SysFunction.cpp b/src/jrd/SysFunction.cpp index 068fe65bb2..52d1b408d5 100644 --- a/src/jrd/SysFunction.cpp +++ b/src/jrd/SysFunction.cpp @@ -223,6 +223,7 @@ bool dscHasData(const dsc* param); // specific setParams functions void setParamsAsciiVal(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, int argsCount, dsc** args); void setParamsBin(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, int argsCount, dsc** args); +void setParamsBlobAppend(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, int argsCount, dsc** args); void setParamsCharToUuid(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, int argsCount, dsc** args); void setParamsDateAdd(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, int argsCount, dsc** args); void setParamsDateDiff(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, int argsCount, dsc** args); @@ -258,6 +259,7 @@ void makeAbs(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* r void makeAsciiChar(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args); void makeBin(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args); void makeBinShift(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args); +void makeBlobAppend(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args); void makeCeilFloor(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args); void makeDateAdd(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args); void makeDateDiff(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args); @@ -297,6 +299,7 @@ dsc* evlAsciiVal(thread_db* tdbb, const SysFunction* function, const NestValueAr dsc* evlAtan2(thread_db* tdbb, const SysFunction* function, const NestValueArray& args, impure_value* impure); dsc* evlBin(thread_db* tdbb, const SysFunction* function, const NestValueArray& args, impure_value* impure); dsc* evlBinShift(thread_db* tdbb, const SysFunction* function, const NestValueArray& args, impure_value* impure); +dsc* evlBlobAppend(thread_db* tdbb, const SysFunction* function, const NestValueArray& args, impure_value* impure); dsc* evlCeil(thread_db* tdbb, const SysFunction* function, const NestValueArray& args, impure_value* impure); dsc* evlCharToUuid(thread_db* tdbb, const SysFunction* function, const NestValueArray& args, impure_value* impure); dsc* evlDateAdd(thread_db* tdbb, const SysFunction* function, const NestValueArray& args, impure_value* impure); @@ -609,6 +612,19 @@ void setParamsAsciiVal(DataTypeUtilBase*, const SysFunction*, int argsCount, dsc } +void setParamsBlobAppend(DataTypeUtilBase*, const SysFunction*, int argsCount, dsc** args) +{ + if (argsCount >= 1 && args[0]->isUnknown()) + args[0]->makeBlob(isc_blob_text, CS_dynamic); + + for (int i = 1; i < argsCount; ++i) + { + if (args[i]->isUnknown()) + args[i]->makeVarying(80, args[0]->getTextType()); + } +} + + void setParamsCharToUuid(DataTypeUtilBase*, const SysFunction*, int argsCount, dsc** args) { if (argsCount >= 1 && args[0]->isUnknown()) @@ -1230,6 +1246,18 @@ void makeBinShift(DataTypeUtilBase*, const SysFunction* function, dsc* result, } +void makeBlobAppend(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, + int argsCount, const dsc** args) +{ + USHORT ttype = CS_dynamic; + + if (argsCount > 0 && args[0]) + ttype = args[0]->getTextType(); + + result->makeBlob(isc_blob_text, ttype); +} + + void makeCeilFloor(DataTypeUtilBase*, const SysFunction* function, dsc* result, int argsCount, const dsc** args) { @@ -2258,6 +2286,146 @@ HUGEINT getScale(impure_value* impure) } +static void appendFromBlob(thread_db* tdbb, jrd_tra* transaction, blb* blob, + const dsc* blobDsc, const dsc* srcDsc) +{ + if (!srcDsc->dsc_address) + return; + + bid* srcBlobID = (bid*)srcDsc->dsc_address; + if (srcBlobID->isEmpty()) + return; + + if (memcmp(blobDsc->dsc_address, srcDsc->dsc_address, sizeof(bid)) == 0) + status_exception::raise(Arg::Gds(isc_random) << Arg::Str("Can not append blob to itself")); + + UCharBuffer bpb; + BLB_gen_bpb_from_descs(srcDsc, blobDsc, bpb); + + AutoBlb srcBlob(tdbb, blb::open2(tdbb, transaction, srcBlobID, bpb.getCount(), bpb.begin())); + + Database* dbb = tdbb->getDatabase(); + + HalfStaticArray buffer; + const SLONG buffSize = (srcBlob->getLevel() == 0) ? + MAX(BUFFER_LARGE, srcBlob->blb_length) : dbb->dbb_page_size - BLP_SIZE; + + UCHAR* buff = buffer.getBuffer(buffSize); + while (!(srcBlob->blb_flags & BLB_eof)) + { + const SLONG len = srcBlob->BLB_get_data(tdbb, buff, buffSize, false); + if (len) + blob->BLB_put_data(tdbb, buff, len); + } +} + + +dsc* evlBlobAppend(thread_db* tdbb, const SysFunction* function, const NestValueArray& args, + impure_value* impure) +{ + Request* request = tdbb->getRequest(); + jrd_tra* transaction = request ? request->req_transaction : tdbb->getTransaction(); + transaction = transaction->getOuter(); + + USHORT ttype = tdbb->getCharSet(); + + blb* blob = NULL; + bid blob_id; + dsc blobDsc; + + blob_id.clear(); + blobDsc.clear(); + + const dsc* argDsc = EVL_expr(tdbb, request, args[0]); + const bool arg0_null = (request->req_flags & req_null) || (argDsc == NULL); + + if (!arg0_null && argDsc->isBlob()) + blob_id = *reinterpret_cast(argDsc->dsc_address); + + const dsc* declDsc = argDsc; + if (!declDsc) + declDsc = EVL_assign_to(tdbb, args[0]); + + if (declDsc && declDsc->isBlob()) + { + ttype = declDsc->getTextType(); + blobDsc.makeBlob(declDsc->getBlobSubType(), ttype, (ISC_QUAD*)&blob_id); + } + else + { + if (declDsc && declDsc->isText()) + ttype = declDsc->getTextType(); + + blobDsc.makeBlob(isc_blob_text, ttype, (ISC_QUAD*) &blob_id); + } + + bool copyBlob = !blob_id.isEmpty(); + if (copyBlob) + { + if (!blob_id.bid_internal.bid_relation_id) + { + if (!transaction->tra_blobs->locate(blob_id.bid_temp_id())) + status_exception::raise(Arg::Gds(isc_bad_segstr_id)); + + BlobIndex blobIdx = transaction->tra_blobs->current(); + if (!blobIdx.bli_materialized && (blobIdx.bli_blob_object->blb_flags & BLB_close_on_read)) + { + blob = blobIdx.bli_blob_object; + copyBlob = false; + } + } + } + + if (!blob) + { + UCharBuffer bpb; + BLB_gen_bpb_from_descs(&blobDsc, &blobDsc, bpb); + bpb.push(isc_bpb_storage); + bpb.push(1); + bpb.push(isc_bpb_storage_temp); + + blob = blb::create2(tdbb, transaction, &blob_id, bpb.getCount(), bpb.begin()); + blob->blb_flags |= BLB_stream | BLB_close_on_read; + } + +// if (copyBlob && argDsc && argDsc->isBlob()) +// appendFromBlob(tdbb, transaction, blob, &blobDsc, argDsc); + + EVL_make_value(tdbb, &blobDsc, impure); + + for (FB_SIZE_T i = 0; i < args.getCount(); i++) + { + if (i == 0) + { + if (arg0_null || argDsc->isBlob() && !copyBlob) + continue; + } + else + { + argDsc = EVL_expr(tdbb, request, args[i]); + if ((request->req_flags & req_null) || !argDsc) + continue; + } + + if (!argDsc->isBlob()) + { + MoveBuffer temp; + UCHAR* addr = NULL; + SLONG len = MOV_make_string2(tdbb, argDsc, ttype, &addr, temp); + + if (addr) + blob->BLB_put_data(tdbb, addr, len); + } + else + { + appendFromBlob(tdbb, transaction, blob, &blobDsc, argDsc); + } + } + + return &impure->vlu_desc; +} + + dsc* evlCeil(thread_db* tdbb, const SysFunction*, const NestValueArray& args, impure_value* impure) { @@ -6593,6 +6761,7 @@ const SysFunction SysFunction::functions[] = {"BIN_SHL_ROT", 2, 2, setParamsInteger, makeBinShift, evlBinShift, (void*) funBinShlRot}, {"BIN_SHR_ROT", 2, 2, setParamsInteger, makeBinShift, evlBinShift, (void*) funBinShrRot}, {"BIN_XOR", 2, -1, setParamsBin, makeBin, evlBin, (void*) funBinXor}, + {"BLOB_APPEND", 2, -1, setParamsBlobAppend, makeBlobAppend, evlBlobAppend, NULL}, {"CEIL", 1, 1, setParamsDblDec, makeCeilFloor, evlCeil, NULL}, {"CEILING", 1, 1, setParamsDblDec, makeCeilFloor, evlCeil, NULL}, {"CHAR_TO_UUID", 1, 1, setParamsCharToUuid, makeUuid, evlCharToUuid, NULL}, diff --git a/src/jrd/blb.cpp b/src/jrd/blb.cpp index d0abd0823c..e034c4d3e4 100644 --- a/src/jrd/blb.cpp +++ b/src/jrd/blb.cpp @@ -59,6 +59,7 @@ #include "../jrd/dpm_proto.h" #include "../jrd/err_proto.h" #include "../jrd/evl_proto.h" +#include "../jrd/exe_proto.h" #include "../jrd/filte_proto.h" #include "../yvalve/gds_proto.h" #include "../jrd/intl_proto.h" @@ -113,7 +114,12 @@ void blb::BLB_cancel(thread_db* tdbb) // Release filter control resources if (blb_flags & BLB_temporary) + { + if (!(blb_flags & BLB_closed)) + blb_transaction->tra_temp_blobs_count--; + delete_blob(tdbb, 0); + } destroy(true); } @@ -189,11 +195,14 @@ bool blb::BLB_close(thread_db* tdbb) SET_TDBB(tdbb); + const bool alreadyClosed = (blb_flags & BLB_closed); + // Release filter control resources if (blb_filter) BLF_close_blob(tdbb, &blb_filter); + blb_flags &= ~BLB_close_on_read; blb_flags |= BLB_closed; if (!(blb_flags & BLB_temporary)) @@ -202,6 +211,9 @@ bool blb::BLB_close(thread_db* tdbb) return true; } + if (!alreadyClosed) + blb_transaction->tra_temp_blobs_count--; + if (blb_level == 0) { //Database* dbb = tdbb->getDatabase(); @@ -267,6 +279,40 @@ blb* blb::create2(thread_db* tdbb, Database* dbb = tdbb->getDatabase(); CHECK_DBB(dbb); + const int maxTempBlobs = MAX_TEMP_BLOBS; + if (maxTempBlobs > 0 && transaction->tra_temp_blobs_count >= maxTempBlobs) + { + const Request* request = tdbb->getRequest(); + string info; + + if (userBlob) + { + Attachment* att = tdbb->getAttachment(); + info = "By user application"; + if (att->att_remote_process.hasData()) + { + info += string(" (") + att->att_remote_process.c_str() + ")"; + } + } + else if (request) + { + const Statement* const statement = request->getStatement(); + if (statement && statement->sqlText) + info = string("By query: ") + *statement->sqlText; + + string stack; + if (EXE_get_stack_trace(request, stack)) + { + info += "\n"; + info += stack; + } + } + + gds__log("Too many temporary blobs (%i allowed)\n%s", maxTempBlobs, info.c_str()); + + ERR_post(Arg::Gds(isc_random) << Arg::Str("Too many temporary blobs")); + } + // Create a blob large enough to hold a single data page SSHORT from, to; SSHORT from_charset, to_charset; @@ -322,6 +368,7 @@ blb* blb::create2(thread_db* tdbb, blob->blb_space_remaining = blob->blb_clump_size; blob->blb_flags |= BLB_temporary; + blob->blb_transaction->tra_temp_blobs_count++; if (filter_required) { @@ -1162,7 +1209,10 @@ void blb::move(thread_db* tdbb, dsc* from_desc, dsc* to_desc, if (!blob || !(blob->blb_flags & BLB_closed)) { - ERR_post(Arg::Gds(isc_bad_segstr_id)); + if (blob && (blob->blb_flags & BLB_close_on_read)) + blob->BLB_close(tdbb); + else + ERR_post(Arg::Gds(isc_bad_segstr_id)); } if (blob->blb_level && (blob->blb_pg_space_id != relPages->rel_pg_space_id)) @@ -1329,7 +1379,7 @@ blb* blb::open2(thread_db* tdbb, */ // Search the index of transaction blobs for a match - const blb* new_blob = NULL; + blb* new_blob = NULL; if (transaction->tra_blobs->locate(blobId.bid_temp_id())) { current = &transaction->tra_blobs->current(); @@ -1344,7 +1394,10 @@ blb* blb::open2(thread_db* tdbb, if (!new_blob || !(new_blob->blb_flags & BLB_temporary) || !(new_blob->blb_flags & BLB_closed)) { - ERR_post(Arg::Gds(isc_bad_segstr_id)); + if (new_blob && (new_blob->blb_flags & BLB_close_on_read)) + new_blob->BLB_close(tdbb); + else + ERR_post(Arg::Gds(isc_bad_segstr_id)); } blob->blb_lead_page = new_blob->blb_lead_page; @@ -1544,7 +1597,7 @@ void blb::BLB_put_segment(thread_db* tdbb, const void* seg, USHORT segment_lengt // Make sure blob is a temporary blob. If not, complain bitterly. - if (!(blb_flags & BLB_temporary)) + if (!(blb_flags & BLB_temporary) || (blb_flags & BLB_closed)) ERR_post(Arg::Gds(isc_cannot_update_old_blob)); if (blb_filter) diff --git a/src/jrd/blb.h b/src/jrd/blb.h index 38ff245a8e..26176b5e69 100644 --- a/src/jrd/blb.h +++ b/src/jrd/blb.h @@ -179,7 +179,8 @@ const int BLB_closed = 8; // Temporary blob has been closed const int BLB_damaged = 16; // Blob is busted const int BLB_seek = 32; // Seek is pending const int BLB_large_scan = 64; // Blob is larger than page buffer cache -const int BLB_bulk = 128; // Blob created by bulk insert operation +const int BLB_close_on_read = 128; // Temporary blob is not closed until read +const int BLB_bulk = 256; // Blob created by bulk insert operation /* Blob levels are: diff --git a/src/jrd/exe.cpp b/src/jrd/exe.cpp index 7ddc1bae38..be444e1b63 100644 --- a/src/jrd/exe.cpp +++ b/src/jrd/exe.cpp @@ -687,6 +687,12 @@ void EXE_receive(thread_db* tdbb, current->bli_request->req_blobs.fastRemove(); current->bli_request = NULL; } + + if (!current->bli_materialized && + (current->bli_blob_object->blb_flags & BLB_close_on_read)) + { + current->bli_blob_object->BLB_close(tdbb); + } } else { @@ -1270,10 +1276,9 @@ void EXE_execute_triggers(thread_db* tdbb, } -static void stuff_stack_trace(const Request* request) +bool EXE_get_stack_trace(const Request* request, string& sTrace) { - string sTrace; - + sTrace = ""; for (const Request* req = request; req; req = req->req_caller) { const Statement* const statement = req->getStatement(); @@ -1329,7 +1334,14 @@ static void stuff_stack_trace(const Request* request) } } - if (sTrace.hasData()) + return sTrace.hasData(); +} + +static void stuff_stack_trace(const Request* request) +{ + string sTrace; + + if (EXE_get_stack_trace(request, sTrace)) ERR_post_nothrow(Arg::Gds(isc_stack_trace) << Arg::Str(sTrace)); } diff --git a/src/jrd/exe_proto.h b/src/jrd/exe_proto.h index e644e97bfe..bee2754807 100644 --- a/src/jrd/exe_proto.h +++ b/src/jrd/exe_proto.h @@ -40,6 +40,8 @@ void EXE_assignment(Jrd::thread_db* tdbb, const Jrd::ValueExprNode* to, dsc* fro void EXE_execute_db_triggers(Jrd::thread_db*, Jrd::jrd_tra*, enum TriggerAction); void EXE_execute_ddl_triggers(Jrd::thread_db* tdbb, Jrd::jrd_tra* transaction, bool preTriggers, int action); +bool EXE_get_stack_trace(const Jrd::Request* request, Firebird::string& sTrace); + const Jrd::StmtNode* EXE_looper(Jrd::thread_db* tdbb, Jrd::Request* request, const Jrd::StmtNode* in_node); diff --git a/src/jrd/tra.cpp b/src/jrd/tra.cpp index 44e6e55fc0..fd713b1d95 100644 --- a/src/jrd/tra.cpp +++ b/src/jrd/tra.cpp @@ -1234,6 +1234,8 @@ void TRA_release_transaction(thread_db* tdbb, jrd_tra* transaction, Jrd::TraceTr blb::release_array(transaction->tra_arrays); } + fb_assert(transaction->tra_temp_blobs_count == 0); + if (transaction->tra_pool) { // Iterate the doubly linked list of requests for transaction and null out the transaction references diff --git a/src/jrd/tra.h b/src/jrd/tra.h index 1f705424b4..34610325db 100644 --- a/src/jrd/tra.h +++ b/src/jrd/tra.h @@ -149,6 +149,7 @@ typedef Firebird::GenericMap > const int DEFAULT_LOCK_TIMEOUT = -1; // infinite const char* const TRA_BLOB_SPACE = "fb_blob_"; const char* const TRA_UNDO_SPACE = "fb_undo_"; +const int MAX_TEMP_BLOBS = 1000; class jrd_tra : public pool_alloc { @@ -285,6 +286,7 @@ public: UCHAR tra_callback_count; // callback count for 'execute statement' SSHORT tra_lock_timeout; // in seconds, -1 means infinite, 0 means NOWAIT ULONG tra_next_blob_id; // ID of the previous blob or array created in this transaction + ULONG tra_temp_blobs_count; // Number of active temporary blobs const ISC_TIMESTAMP_TZ tra_timestamp; // transaction start time Request* tra_requests; // Doubly linked list of requests active in this transaction MonitoringSnapshot* tra_mon_snapshot; // Database state snapshot (for monitoring purposes)