From f53c23c17a8a72972d487fc2b3a5840a0894025f Mon Sep 17 00:00:00 2001 From: Alexander Peshkov Date: Mon, 23 Oct 2017 17:10:49 +0300 Subject: [PATCH] New interface Batch helping to efficiently implement JDBC prepared statement batches (#99) Batch interface implementation --- builds/posix/firebird.vers | 3 + builds/win32/msvc10/common.vcxproj | 1 + builds/win32/msvc10/common.vcxproj.filters | 19 +- builds/win32/msvc10/engine.vcxproj | 2 + builds/win32/msvc10/engine.vcxproj.filters | 9 + builds/win32/msvc12/common.vcxproj | 1 + builds/win32/msvc12/common.vcxproj.filters | 3 + builds/win32/msvc12/engine.vcxproj | 2 + builds/win32/msvc12/engine.vcxproj.filters | 6 + builds/win32/msvc14/common.vcxproj | 1 + builds/win32/msvc14/common.vcxproj.filters | 3 + builds/win32/msvc14/engine.vcxproj | 2 + builds/win32/msvc14/engine.vcxproj.filters | 6 + doc/Using_OO_API.html | 555 +++++++++++- examples/interfaces/11.batch.cpp | 510 +++++++++++ src/common/MsgMetadata.cpp | 21 +- src/common/MsgMetadata.h | 24 +- src/common/classes/BatchCompletionState.h | 185 ++++ src/common/classes/stack.h | 4 +- src/common/classes/vector.h | 32 +- src/common/config/config.cpp | 9 +- src/common/config/config.h | 3 + src/common/utils.cpp | 21 +- src/common/utils_proto.h | 5 +- src/dsql/DdlNodes.epp | 2 +- src/dsql/DsqlBatch.cpp | 988 +++++++++++++++++++++ src/dsql/DsqlBatch.h | 160 ++++ src/dsql/StmtNodes.cpp | 14 +- src/dsql/StmtNodes.h | 4 +- src/dsql/dsql.cpp | 72 +- src/dsql/dsql.h | 7 +- src/dsql/gen.cpp | 2 +- src/include/firebird/FirebirdInterface.idl | 73 ++ src/include/firebird/IdlFbInterfaces.h | 650 +++++++++++++- src/jrd/Attachment.h | 2 +- src/jrd/EngineInterface.h | 44 + src/jrd/SysFunction.cpp | 24 +- src/jrd/blr.h | 3 +- src/jrd/ibase.h | 4 + src/jrd/inf_pub.h | 1 + src/jrd/jrd.cpp | 402 ++++++++- src/jrd/jrd_proto.h | 2 + src/jrd/req.h | 1 + src/misc/codes.epp | 10 +- src/remote/client/interface.cpp | 927 ++++++++++++++++++- src/remote/protocol.cpp | 547 +++++++++++- src/remote/protocol.h | 74 ++ src/remote/remote.h | 50 +- src/remote/server/server.cpp | 259 ++++++ src/yvalve/YObjects.h | 33 + src/yvalve/utl.cpp | 10 +- src/yvalve/why.cpp | 283 +++++- src/yvalve/why_proto.h | 1 + 53 files changed, 5958 insertions(+), 118 deletions(-) create mode 100644 examples/interfaces/11.batch.cpp create mode 100644 src/common/classes/BatchCompletionState.h create mode 100644 src/dsql/DsqlBatch.cpp create mode 100644 src/dsql/DsqlBatch.h diff --git a/builds/posix/firebird.vers b/builds/posix/firebird.vers index 5137b6f55a..54349c4dba 100644 --- a/builds/posix/firebird.vers +++ b/builds/posix/firebird.vers @@ -343,8 +343,11 @@ fb_ping fb_get_master_interface +# Legacy handles translation + fb_get_database_handle fb_get_transaction_handle +#fb_get_statement_interface fb_database_crypt_callback fb_dsql_set_timeout diff --git a/builds/win32/msvc10/common.vcxproj b/builds/win32/msvc10/common.vcxproj index 930d78f471..6f9e926acb 100644 --- a/builds/win32/msvc10/common.vcxproj +++ b/builds/win32/msvc10/common.vcxproj @@ -114,6 +114,7 @@ + diff --git a/builds/win32/msvc10/common.vcxproj.filters b/builds/win32/msvc10/common.vcxproj.filters index 1463f617a2..feac0f28d1 100644 --- a/builds/win32/msvc10/common.vcxproj.filters +++ b/builds/win32/msvc10/common.vcxproj.filters @@ -222,21 +222,6 @@ classes - - tomcrypt - - - tomcrypt - - - tomcrypt - - - tomcrypt - - - tomcrypt - classes @@ -557,6 +542,9 @@ headers +<<<<<<< HEAD + +======= headers @@ -594,6 +582,7 @@ headers +>>>>>>> master headers diff --git a/builds/win32/msvc10/engine.vcxproj b/builds/win32/msvc10/engine.vcxproj index 75c3ea8b5b..37fa7e2738 100644 --- a/builds/win32/msvc10/engine.vcxproj +++ b/builds/win32/msvc10/engine.vcxproj @@ -37,6 +37,7 @@ + @@ -168,6 +169,7 @@ + diff --git a/builds/win32/msvc10/engine.vcxproj.filters b/builds/win32/msvc10/engine.vcxproj.filters index e55b9edd0f..4b40cbe048 100644 --- a/builds/win32/msvc10/engine.vcxproj.filters +++ b/builds/win32/msvc10/engine.vcxproj.filters @@ -459,6 +459,12 @@ JRD files\EXTDS + + JRD files + + + DSQL + @@ -986,6 +992,9 @@ Header files + + Header files + diff --git a/builds/win32/msvc12/common.vcxproj b/builds/win32/msvc12/common.vcxproj index a642e6e881..ded46c0b63 100644 --- a/builds/win32/msvc12/common.vcxproj +++ b/builds/win32/msvc12/common.vcxproj @@ -105,6 +105,7 @@ + diff --git a/builds/win32/msvc12/common.vcxproj.filters b/builds/win32/msvc12/common.vcxproj.filters index 7b8689c4f1..3d0ba8081a 100644 --- a/builds/win32/msvc12/common.vcxproj.filters +++ b/builds/win32/msvc12/common.vcxproj.filters @@ -539,5 +539,8 @@ headers + + headers + \ No newline at end of file diff --git a/builds/win32/msvc12/engine.vcxproj b/builds/win32/msvc12/engine.vcxproj index e722c09a81..37f53c9375 100644 --- a/builds/win32/msvc12/engine.vcxproj +++ b/builds/win32/msvc12/engine.vcxproj @@ -37,6 +37,7 @@ + @@ -168,6 +169,7 @@ + diff --git a/builds/win32/msvc12/engine.vcxproj.filters b/builds/win32/msvc12/engine.vcxproj.filters index 2021ebded7..4a23bff170 100644 --- a/builds/win32/msvc12/engine.vcxproj.filters +++ b/builds/win32/msvc12/engine.vcxproj.filters @@ -462,6 +462,9 @@ JRD files + + DSQL + @@ -989,6 +992,9 @@ Header files + + Header files + diff --git a/builds/win32/msvc14/common.vcxproj b/builds/win32/msvc14/common.vcxproj index aa5ed698b0..5b8841f6fc 100644 --- a/builds/win32/msvc14/common.vcxproj +++ b/builds/win32/msvc14/common.vcxproj @@ -105,6 +105,7 @@ + diff --git a/builds/win32/msvc14/common.vcxproj.filters b/builds/win32/msvc14/common.vcxproj.filters index 7b8689c4f1..3d0ba8081a 100644 --- a/builds/win32/msvc14/common.vcxproj.filters +++ b/builds/win32/msvc14/common.vcxproj.filters @@ -539,5 +539,8 @@ headers + + headers + \ No newline at end of file diff --git a/builds/win32/msvc14/engine.vcxproj b/builds/win32/msvc14/engine.vcxproj index ac9311afb2..b0ff419b68 100644 --- a/builds/win32/msvc14/engine.vcxproj +++ b/builds/win32/msvc14/engine.vcxproj @@ -37,6 +37,7 @@ + @@ -168,6 +169,7 @@ + diff --git a/builds/win32/msvc14/engine.vcxproj.filters b/builds/win32/msvc14/engine.vcxproj.filters index 2021ebded7..4a23bff170 100644 --- a/builds/win32/msvc14/engine.vcxproj.filters +++ b/builds/win32/msvc14/engine.vcxproj.filters @@ -462,6 +462,9 @@ JRD files + + DSQL + @@ -989,6 +992,9 @@ Header files + + Header files + diff --git a/doc/Using_OO_API.html b/doc/Using_OO_API.html index 12c7a3f674..e2958641e0 100644 --- a/doc/Using_OO_API.html +++ b/doc/Using_OO_API.html @@ -6,6 +6,15 @@ + + + + + + + + + @@ -35,8 +44,10 @@ independent – that means that to define/use them one need not use language specific constructions like class in C++, interface may be -defined using any language having concepts of array and pointer to -procedure/function. Next interfaces are versioned +defined using any language able +to call functions using C calling conventions and having +concepts of array and pointer to procedure/function. Next interfaces +are versioned – i.e. we support different versions of same interface. Binary layout of interfaces is designed to support that features very efficient (there is no need in @@ -637,6 +648,9 @@ the following:

FB_BOOLEAN

FB_CHAR(len)

FB_DATE

+

FB_DECFIXED(scale)

+

FB_DECFLOAT16

+

FB_DECFLOAT34

FB_DOUBLE

FB_FLOAT

FB_INTEGER

@@ -644,9 +658,9 @@ the following:

charSet)

FB_INTL_VARCHAR(len, charSet)

-

FB_SCALED_BIGINT(x)

-

FB_SCALED_INTEGER(x)

-

FB_SCALED_SMALLINT(x)

+

FB_SCALED_BIGINT(scale)

+

FB_SCALED_INTEGER(scale)

+

FB_SCALED_SMALLINT(scale)

FB_SMALLINT

FB_TIME

FB_TIMESTAMP

@@ -829,10 +843,322 @@ completion code that function is notified (by passing false as last parameter) that segment was not read completely and continuation is expected at next call.

After -finishing with blob do not forget top close it:

+finishing with blob do not forget to close it:

blob->close(&status);


+

+

Modifying +data in a batch.

+

Since +version 4 firebird supports batch execution of statements with input +parameters – that means sending more than single set of parameters +when executing statement. Batch interface is +designed (first of all) in order to satisfy JDBC requirements for +prepared statement’s batch processing but has some serious +differences:

+

- +like all operations with data in firebird it’s oriented on +messages, not single field;

+

- +as an important extension out batch interface supports inline use of +blobs (specially efficient when working with small blobs);

+

- +execute() method returns not plain array of integers but special +BatchCompletionState interface +which can (depending upon batch creation parameters) contain both +update records info and in addition to error flag detailed status +vectors for messages that caused execution errors.

+


+ +

+

Batch +(exactly like ResultSet) may be created in 2 +ways – using Statement or Attachment +interface, in both cases createBatch() method of appropriate +interface is called. In second case text of SQL statement to be +executed in a batch is passed directly to createBatch(). Tuning of +batch operation is performed using Batch +parameters block which has format more or less similar to DPB v.2 +– tag in the beginning (IBatch::CURRENT_VERSION) followed by the +set of wide clumplets: 1-byte tag, 4-byte length, length-byte value. +Possible tags are described in batch +interface. The simplest (and recommended) way to create parameters +block for batch creation is to use appropriate XpbBuilder +interface:

+

IXpbBuilder* +pb = utl->getXpbBuilder(&status, IXpbBuilder::BATCH, NULL, 0);

+

pb->insertInt(&status, +IBatch::RECORD_COUNTS, 1);

+

Use +of such parameters block directs batch to account number of updated +records on per-message basis.

+

To +create batch interface with desired parameters pass parameters block +to createBatch() call:

+

IBatch* +batch = att->createBatch(&status, tra, 0, sqlStmtText, +SQL_DIALECT_V6, NULL,

+

pb->getBufferLength(&status), +pb->getBuffer(&status));

+

In +this sample batch interface is created with default format of +messages cause NULL is passed instead input metadata format.

+


+ +

+

In +order to proceed with created batch interface we need to know format +of messages in it. It can be obtained using getMetadata() method:

+

IMessageMetadata* +meta = batch->getMetadata(&status);

+

Certainly +if you have passed your own format of messages to the batch you may +simply use it.

+

+In the former text I suppose +that some function fillNextMessage(unsigned char* data, +IMessageMetadata* metadata) is present and can fill buffer ‘data’ +according to passed format ‘metadata’. In order to work with +messages we need a buffer for a data:

+

unsigned +char* data = new unsigned char[meta->getMessageLength(&status)];

+

+Now we can add some messages +full of data to the batch:

+

fillNextMessage(data, +meta);

+

batch->add(&status, +1, data);

+

fillNextMessage(data, +meta);

+

batch->add(&status, +1, data);

+

+An alternative way of working +with messages (using FB_MESSAGE macro) is present in the sample of +using batch interface 11.batch.cpp.

+


+ +

+

+Finally batch should be +executed:

+

IBatchCompletionState* +cs = batch->execute(&status, tra);

+

We +requested accounting of the number of modified (inserted, updated or +deleted) records per message. To print it we must use +BatchCompletionState interface. +Determine total number of messages processed by batch (it can be less +than the number of messages passed to the batch if error happened and +an option enabling multiple errors during batch processing was not +turned on):

+

unsigned +total = cs->getSize(&status);

+

Now print the state of each +message:

+

for +(unsigned p = 0; p < total; ++p) printf(“Msg %u state %d\n”, +p, cs->getState(&status, p));

+

When +finished analyzing completion state don’t forget to dispose it:

+

cs->dispose();

+

Full +sample of printing contents +of BatchCompletionState +is in print_cs() function in sample 11.batch.cpp.

+

If +for some reason you want to make batch buffers empty not executing it +(i.e. prepare for new portion of messages to process) use cancel() +method:

+

batch->cancel(&status);

+

+Being +reference counted Batch does not have special method to close it – +standard release() call:

+

batch->release();

+

+Described +methods help to implement all what one needs for JDBC-style prepared +statement batch operations. +

+


+
+ +

+

One can add more than +single message in one call to the batch. When doing it please +remember – messages should be appropriately aligned for this +feature to work correctly. Required alignment and aligned size of the +message should be obtained from MessageMetadata +interface, for example:

+

unsigned aligned = +meta->getAlignedLength(&status);

+

Later that size will be +useful when allocating an array of messages and working with it:

+

unsigned char* data = +new unsigned char[aligned * N]; // N is desired number of messages

+

for (int n = 0; n < +N; ++n) fillNextMessage(&data[aligned * n], meta);

+

batch->add(&status, +N, data);

+

After +it batch may be executed or next portion of messages added to it.

+


+
+ +

+

Blobs +in general are not compatible with batches – batch is efficient +when one needs to pass a lot of small data to the server in single +step, blobs are treated as large objects and therefore in general it +makes no sense to use them in batches. But on practice it often +happens that blobs are not too big – and in this case use of +traditional blob API (create blob, pass segments to the server, close +blob, pass blobs ID in the message) kills performance, specially when +used over WAN. Therefore in firebird batch supports passing blobs to +server inline, together with other messages. To use that feature +first of all blob usage policy for a +batch to be created should be set (as an option in parameters +block):

+

pb->insertInt(&status, +IBatch::BLOB_IDS, IBatch::BLOB_IDS_ENGINE);

+

In +this example temporal blob IDs needed to keep a link between blob and +a message where it’s used will be generated by firebird engine – +that’s the simplest and rather common usage. Imagine that the +message is described as follows:

+

FB_MESSAGE(Msg, +ThrowStatusWrapper,

+

(FB_VARCHAR(5), id)

+

(FB_VARCHAR(10), name)

+

(FB_BLOB, desc)

+

) project(&status, +master);

+

In +that case to send a message containing blob to the server one can do +something like this:

+

project->id = +++idCounter;

+

project->name.set(currentName);

+

batch->addBlob(&status, +descriptionSize, descriptionText, &project->desc);

+

batch->add(&status, +1, project.getData());

+

+If some blob happened to be +big enough not to fit into your existing buffer you may instead +reallocating buffer use appendBlobData() method. It appends more data +to last added blob. +

+

batch->addBlob(&status, +descriptionSize, descriptionText, &project→desc, bpbLength, +bpb);

+

After +adding first part of blob get next portion of data into +descriptionText, update descriptionSize and:

+

batch->appendBlobData(&status, +descriptionSize, descriptionText);

+

This +may be done in a loop but take care not to overflow internal batch +buffers – it’s size is controlled by BUFFER_BYTES_SIZE +option when creating batch interface but can’t exceed 40Mb (default +is 10Mb). If you need to process such big blob (for example on the +background of a lot of small one – this can explain use of batch) +just use standard blob API and registerBlob +method of Batch interface.

+

One +more possible choice of blob policy is BLOB_IDS_USER. Usage at the +first look does not change much – before calling addBlob() correct +and unique per batch execution ID should be placed to the memory +referenced by last parameter. Certainly same ID should be passed in +the data message to the blob. Taking into an account that generation +of blob IDs by engine is very fast such policy may seem useless but +imagine a case when you get blobs and other data in relatively +independent streams (blocks in a file for example) and some good IDs +are already present in them. In such case use of user-supplied blob +IDs can greatly simplify your code.

+

Please +take into an account – unlike blobs created using regular +createBlob() blobs created by Batch +interface are by default stream, not segmented. Segmented blobs +provide nothing interesting compared with stream one and therefore +not recommended to be used in new development, we support that format +only for backward compatibility reasons. If you really need segmented +blobs this default may be overridden by calling:

+

batch->setDefaultBpb(&status, +bpbLength, +bpb);

+

Certainly +passed BPB may contain any other blob creation parameters too. As you +may have already noticed you may also pass BPB directly to addBlob() +but if most of blobs you are going to add have same non-default +format use of setDefaultBpb() is slightly more efficient. Returning +to segmented blobs – call to addBlob() will add first segment to +the blob, following calls to appendBlobData() will add more segments. +Do not forget that segment size is limited to 64Kb – 1, an attempt +to pass more data in a single call with cause an error.

+

Next +step when working with existing blob streams is use of +addBlobStream() method. Using it one can add more than one blob to +the batch per single call. Blob stream is a sequence of blobs, each +starts with blob header. Header should be appropriately aligned - +Batch interface provides special call for this +purpose:

+

unsigned +alignment = batch->getBlobAlignment(&status);

+

It’s +supposed that all components of blob stream in a batch should be +aligned at least at alignment boundary, including size of stream +potions passed to addBlobStream() which should be a multiple of this +alignment. Header contains 3 fields – 8-byte blob ID (must be +non-zero), 4-byte total blob size and 4 byte BPB size. Total blob +size includes BPB inside, i.e. one will always find next blob in the +stream in blob-size bytes after the header (taking into account the +alignment). BPB (if present, i.e. if BPB size is not zero) is placed +right after the header. After BPB blob data goes, it’s format +depends upon blob type – stream or segmented. In case of stream +blob it’s a plain sequence of bytes having size blob-size – +BPB-size. With segmented blob things are a bit more compicated: blob +data is a set of segments where each segment has the following format +– 2-bytes size of segment (this should be aligned at +IBatch::BLOB_SEGHDR_ALIGN boundary) followed by stored in this 2 +bytes number of bytes.

+

When +big blob is added to the stream it’s size is not always known in +advance. In order not to have too big buffer for that blob (remember, +size should be provided in blob header, before blob data) blob +continuation record may be used. In blob header you leave blob size +at a value known when creating that header and add continuation +record that has format absolutely same as blob header, but here blob +ID must be zero and BPB size must always be zero too. Typically you +will want to have one continuation record per addBlobStream() call.

+

Last +method used to work with blobs stands alone from the first three that +pass blob data inline with the rest of batch data – +it’s +needed to register in a batch ID of a blob created using standard +blob API. This may be unavoidable if one needs to pass to a batch +really big blob. Do not use ID of such blob in batch directly – +that will cause invalid blob ID error during batch execution. Instead +do:

+

batch->registerBlob(&status, +&realId, &msg->desc);

+

+If blob policy makes firebird +engine generate blob IDs this code is enough to correctly register +existing blob in a batch. In other cases you will have to assign +correct (from batch POV) ID to msg->desc.

+


+ +

+

+Almost all mentioned methods +are used in 11.batch.cpp – please use it to see an alive sample of +batching in firebird.

+


+

Working with events.

Events @@ -1469,6 +1795,16 @@ interface – replaces isc_db_handle:

cursorFlags is needed to open bidirectional cursor setting it's value to Istatement::CURSOR_TYPE_SCROLLABLE.

  • +

    IBatch* + createBatch(StatusType* status, ITransaction* transaction, unsigned + stmtLength, const char* sqlStmt, unsigned dialect, IMessageMetadata* + inMetadata, unsigned parLength, + const unsigned char* par) – prepares sqlStmt and creates Batch + interface ready to accept multiple sets of input parameters in + inMetadata format. Leaving inMetadata NULL + makes batch use default format for sqlStmt. Parameters block may be + passed to createBatch() making it possible to adjust batch behavior.

    +

  • IEvents* queEvents(StatusType* status, IEventCallback* callback, unsigned length, const unsigned char* events) – replaces isc_que_events() @@ -1494,13 +1830,177 @@ interface – replaces isc_db_handle:


    +

    Batch +interface – makes it possible to process multiple sets of +parameters in single statement execution.

    +
      +
    1. +

      void + add(StatusType* status, unsigned count, + const void* inBuffer) – adds count messages from inBuffer to the + batch. Total size of messages that can be added to the batch is + limited by BUFFER_BYTES_SIZE parameter of + batch creation.

      +
    2. +

      void + addBlob(StatusType* status, unsigned length, + const void* inBuffer, ISC_QUAD* blobId, unsigned + bpbLength, const unsigned char* bpb) + – adds single blob having length bytes from inBuffer to the batch, + blob identifier is located at blobId address. If + blob should be created with non-default parameters BPB may be passed + (format matches one used in Attachment::createBlob).Total + size of inline blobs that can be added to the batch (including + optional BPBs, blob headers, segment sizes and taking into an + accoount alignment) is + limited by BUFFER_BYTES_SIZE parameter of + batch creation (affects all blob-oriented methods except + registerBlob()). +

      +
    3. +

      void + appendBlobData(StatusType* status, unsigned length, const void* + inBuffer) – extend last added blob: append length bytes taken from + inBuffer address to it.

      +
    4. +

      void + addBlobStream(StatusType* status, unsigned length, const void* + inBuffer) – adds blob data (this can be multiple objects or part + of single blob) to the batch. Header of each blob in the stream is + aligned at getBlobAlignment() boundary and contains 3 fields: first + - 8-bytes blob identifier (in ISC_QUAD format), second - 4-bytes + length of blob, third – 4-bytes length of BPB. Blob header should + not cross boundaries of buffer in this function call. BPB data is + placed right after header, blob data goes next. Length of blob + includes BPB (if it present). All data may be distributed between + multiple addBlobStream() calls. Blob data in turn may be structured + in case of segmented blob, see chapter “Modifying + data in a batch”.

      +
    5. +

      void + registerBlob(StatusType* status, const ISC_QUAD* existingBlob, + ISC_QUAD* blobId) – makes it possible to use in batch blobs added + using standard Blob interface. This function + contains 2 ISC_QUAD* parameters, it’s important not to mix them – + second parameter (existingBlob) is a pointer to blob identifier, + already added out of batch scope, third (blobId) points to blob + identifier that will be placed in a message in this batch.

      +
    6. +

      IBatchCompletionState* + execute(StatusType* status, ITransaction* transaction) – execute + batch with parameters passed to it in the messages. If parameter + MULTIERROR is not set in parameters block + when creating the batch execution will be stopped after first error, + in MULTIERROR mode an unlimited number of errors can happen, after + an error execution is continued from the next message. This function + returns BatchCompletionState interface that contains all requested + nformation about the results of batch execution.

      +
    7. +

      void + cancel(StatusType* status) – clear messages and blobs buffers, + return batch to a state it had right after creation. Notice – + being reference counted interface batch does not contain any special + function to close it, please use release() for this purposes. +

      +
    8. +

      unsigned + getBlobAlignment(StatusType* status) – returns required alignment + for the data placed into the buffer of addBlobStream().

      +
    9. +

      IMessageMetadata* + getMetadata(StatusType* status) – return format of metadata used + in batch’s messages.

      +
    10. +

      void + setDefaultBpb(StatusType* status, unsigned parLength, const unsigned + char* par) – sets BPB which will be used for all blobs missing + non-default BPB. Must be called before adding any message or blob to + batch.

      +
    +

    Tag +for parameters block:

    +

    VERSION1

    +

    Tags +for clumplets in parameters block:

    +

    MULTIERROR +(0/1) – can have >1 message with errors

    +

    RECORD_COUNTS +(0/1) - per-message modified records accounting

    +

    BUFFER_BYTES_SIZE +(integer) - maximum possible buffer size (default 10Mb, maximum 40Mb)

    +

    BLOB_IDS +- policy used to store blobs

    +

    DETAILED_ERRORS +(integer) - how many vectors with detailed error info are stored in +completion state (default 64, maximum 256)

    +

    Policies +used to store blobs:

    +

    BLOB_IDS_NONE +– inline blobs can't be used (registerBlob() works anyway)

    +

    BLOB_IDS_ENGINE +- blobs are added one by one, IDs are generated by firebird engine

    +

    BLOB_IDS_USER +- blobs are added one by one, IDs are generated by user

    +

    BLOB_IDS_STREAM +- blobs are added in a stream, IDs are generated by user

    +


    + +

    +

    BatchCompletionState +– disposable interface, always returned by execute() method of +Batch interface. It contains more or less +(depending upon parameters passed when Batch was +created) detailed information about the results of batch execution.

    +

    {

    +
      +
    1. +

      uint + getSize(StatusType* status) – returns the total number of + processed messages.

      +
    2. +

      int + getState(StatusType* status, uint pos) – returns the result of + execution of message number ‘pos’. On any error with the message + this is EXECUTE_FAILED constant, value returned on success depends + upon presence of RECORD_COUNTS parameter of + batch creation. When it present and has non-zero value number of + records inserted, updated or deleted during particular message + processing is returned, else SUCCESS_NO_INFO constant is returned.

      +
    3. +

      uint + findError(StatusType* status, uint pos) – finds next (starting + with pos) message which processing caused an error. When such + message is missing NO_MORE_ERRORS constant is returned. Number of + status vectors, returned in this interface, is limited by the value + of DETAILED_ERRORS parameter of batch + creation.

      +
    4. +

      void + getStatus(StatusType* status, IStatus* to, uint pos) – returns + detailed information (full status vector) about an error that took + place when processing ‘pos’ message. In order to distinguish + between errors (in Batch::execute() or in + BatchCompletionState::getStatus()) + that status is returned in separate ‘to’ parameter unlike errors + in this call that are placed into ‘status’ parameter.

      +
    +

    Special +values returned by getState():

    +

    EXECUTE_FAILED +- error happened when processing this message

    +

    SUCCESS_NO_INFO +- record update info was not collected

    +

    Special +value returned by findError():

    +

    NO_MORE_ERRORS +– no more messages with errors in this batch


    Blob interface – replaces isc_blob_handle:

      -
    1. +

    2. void getInfo(StatusType* status, unsigned itemsLength, const unsigned char* items, unsigned bufferLength, unsigned char* buffer) – @@ -1881,7 +2381,17 @@ with execution of SQL statements.

    3. unsigned getMessageLength(StatusType* status) - returns length of message - buffer (use it to allocate memory for the buffer).

      + buffer (use it to allocate memory for the buffer).

      +
    4. +

      unsigned + getAlignment(StatusType* status) – returns alignment required for + message buffer.

      +
    5. +

      unsigned + getAlignedLength(StatusType* + status) – returns length of message buffer taking into an account + alignment requirements (use it to allocate memory for an array of + buffers and navigate through that array).


    @@ -2270,21 +2780,31 @@ interface – replaces (partially) isc_stmt_handle.

    outMetadata, void* outBuffer) – executes any SQL statement except returning multiple rows of data. Partial analogue of isc_dsql_execute2() - in and out XSLQDAs replaced with input and - output messages with appropriate buffers.

    + output messages with appropriate buffers.

  • -

    IResultSet* +

    IResultSet* openCursor(StatusType* status, ITransaction* transaction, IMessageMetadata* inMetadata, void* inBuffer, IMessageMetadata* outMetadata, unsigned flags) – executes SQL statement potentially - returning multiple rows of data. Returns ResultSet interface which - should be used to fetch that data. Format of output data is defined - by outMetadata parameter, leaving it NULL default format may be - used. Parameter flags is needed to open bidirectional cursor setting - it's value to Istatement::CURSOR_TYPE_SCROLLABLE.

    + returning multiple rows of data. Returns ResultSet + interface which should be used to fetch that data. Format of output + data is defined by outMetadata parameter, leaving it NULL default + format may be used. Parameter flags is needed to open bidirectional + cursor setting it's value to Istatement::CURSOR_TYPE_SCROLLABLE.

  • -

    void +

    IBatch* + createBatch(StatusType* status, IMessageMetadata* inMetadata, uint + parLength, const uchar* par) – creates Batch + interface to SQL statement with input parameters making it possible + to execute that statement with multiple sets of parameters. Format + of input data is defined by inMetadata parameter, leaving it NULL + makes batch use default format from this interface. Parameters block + may be passed to createBatch() making it possible to adjust batch + behavior.

    +
  • +

    void setCursorName(StatusType* status, const char* name) – replaces - isc_dsql_set_cursor_name(). + isc_dsql_set_cursor_name().

  • void @@ -2710,6 +3230,7 @@ defined by XpbBuilder interface:

    Valid builder types:

    +

    BATCH

    DPB

    SPB_ATTACH

    SPB_START

    diff --git a/examples/interfaces/11.batch.cpp b/examples/interfaces/11.batch.cpp new file mode 100644 index 0000000000..8d23b12e28 --- /dev/null +++ b/examples/interfaces/11.batch.cpp @@ -0,0 +1,510 @@ +/* + * PROGRAM: Object oriented API samples. + * MODULE: 11.batch.cpp + * DESCRIPTION: A trivial sample of using Batch interface. + * + * Example for the following interfaces: + * IBatch - interface to work with FB pipes + * + * c++ 11.batch.cpp -lfbclient + * + * The contents of this file are subject to the Initial + * Developer's Public License Version 1.0 (the "License"); + * you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * http://www.ibphoenix.com/main.nfs?a=ibphoenix&page=ibp_idpl. + * + * Software distributed under the License is distributed AS IS, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. + * See the License for the specific language governing rights + * and limitations under the License. + * + * The Original Code was created by Alexander Peshkoff + * for the Firebird Open Source RDBMS project. + * + * Copyright (c) 2017 Alexander Peshkoff + * and all contributors signed below. + * + * All Rights Reserved. + * Contributor(s): ______________________________________. + */ + +#include "ifaceExamples.h" +#include + +static IMaster* master = fb_get_master_interface(); + + +// output error message to user + +static void errPrint(IStatus* status) +{ + char buf[256]; + master->getUtilInterface()->formatStatus(buf, sizeof(buf), status); + fprintf(stderr, "%s\n", buf); +} + + +// align target to alignment boundary + +template +static inline T align(T target, uintptr_t alignment) +{ + return (T) ((((uintptr_t) target) + alignment - 1) & ~(alignment - 1)); +} + + +// append given message to buffer ptr + +static void putMsg(unsigned char*& ptr, const void* from, unsigned size, unsigned alignment) +{ + memcpy(ptr, from, size); + ptr += align(size, alignment); +} + + +// append blob header with BPB to buffer ptr +// return pointer to blob size field - prefilled with BPB size + +static unsigned* putBlobHdr(unsigned char*& ptr, unsigned alignment, ISC_QUAD* id, unsigned bpbSize, const unsigned char* bpb) +{ + ptr = align(ptr, alignment); + + memcpy(ptr, id, sizeof(ISC_QUAD)); + ptr += sizeof(ISC_QUAD); + + unsigned* rc = reinterpret_cast(ptr); + + memcpy(ptr, &bpbSize, sizeof(unsigned)); + ptr += sizeof(unsigned); + memcpy(ptr, &bpbSize, sizeof(unsigned)); + ptr += sizeof(unsigned); + + memcpy(ptr, bpb, bpbSize); + ptr += bpbSize; + + return rc; +} + + +// append given blob to buffer ptr + +static void putBlob(unsigned char*& ptr, const void* from, unsigned size, unsigned alignment, ISC_QUAD* id) +{ + unsigned* sizePtr = putBlobHdr(ptr, alignment, id, 0, NULL); + memcpy(ptr, from, size); + *sizePtr += size; + ptr += size; + + ptr = align(ptr, alignment); +} + + +// append given segment to buffer ptr + +unsigned putSegment(unsigned char*& ptr, const char* testData) +{ + ptr = align(ptr, IBatch::BLOB_SEGHDR_ALIGN); + unsigned short l = strlen(testData); + memcpy(ptr, &l, sizeof l); + ptr += sizeof l; + memcpy(ptr, testData, l); + ptr += l; + return align(l + sizeof l, IBatch::BLOB_SEGHDR_ALIGN); +} + +// BatchCompletionState printer - prints all what we know about completed batch + +static void print_cs(ThrowStatusWrapper& status, IBatchCompletionState* cs, IUtil* utl) +{ + unsigned p = 0; + IStatus* s2 = NULL; + bool pr1 = false, pr2 = false; + + // 1. Print per-message state info + + unsigned upcount = cs->getSize(&status); + unsigned unk = 0, succ = 0; + for (p = 0; p < upcount; ++p) + { + int s = cs->getState(&status, p); + switch (s) + { + case IBatchCompletionState::EXECUTE_FAILED: + if (!pr1) + { + printf("Message Status\n", p); + pr1 = true; + } + printf("%5u Execute failed\n", p); + break; + + case IBatchCompletionState::SUCCESS_NO_INFO: + ++unk; + break; + + default: + if (!pr1) + { + printf("Message Status\n", p); + pr1 = true; + } + printf("%5u Updated %d record(s)\n", p, s); + ++succ; + break; + } + } + printf("Summary: total=%u success=%u success(but no update info)=%u\n", upcount, succ, unk); + + // 2. Print detailed errors (if exist) for messages + + s2 = master->getStatus(); + for(p = 0; (p = cs->findError(&status, p)) != IBatchCompletionState::NO_MORE_ERRORS; ++p) + { + try + { + cs->getStatus(&status, s2, p); + + char text[1024]; + utl->formatStatus(text, sizeof(text) - 1, s2); + text[sizeof(text) - 1] = 0; + if (!pr2) + { + printf("\nDetailed errors status:\n", p); + pr2 = true; + } + printf("Message %u: %s\n", p, text); + } + catch (const FbException& error) + { + // handle error + fprintf(stderr, "\nError describing message %u\n", p); + errPrint(error.getStatus()); + fprintf(stderr, "\n"); + } + } + + if (s2) + s2->dispose(); +} + +int main() +{ + int rc = 0; + + // set default password if none specified in environment + setenv("ISC_USER", "sysdba", 0); + setenv("ISC_PASSWORD", "masterkey", 0); + + // With ThrowStatusWrapper passed as status interface FbException will be thrown on error + ThrowStatusWrapper status(master->getStatus()); + + // Declare pointers to required interfaces + IProvider* prov = master->getDispatcher(); + IUtil* utl = master->getUtilInterface(); + IAttachment* att = NULL; + ITransaction* tra = NULL; + IBatch* batch = NULL; + IBatchCompletionState* cs = NULL; + IXpbBuilder* pb = NULL; + + unsigned char streamBuf[10240]; // big enough for demo + unsigned char* stream = NULL; + + try + { + // attach employee db + att = prov->attachDatabase(&status, "employee", 0, NULL); + tra = att->startTransaction(&status, 0, NULL); + + // cleanup + att->execute(&status, tra, 0, "delete from project where proj_id like 'BAT%'", SAMPLES_DIALECT, + NULL, NULL, NULL, NULL); + + // + printf("\nPart 1. Simple messages. Adding one by one or by groups of messages.\n"); + // + + // Message to store in a table + FB_MESSAGE(Msg1, ThrowStatusWrapper, + (FB_VARCHAR(5), id) + (FB_VARCHAR(10), name) + ) project1(&status, master); + project1.clear(); + IMessageMetadata* meta = project1.getMetadata(); + + // sizes & alignments + unsigned mesAlign = meta->getAlignment(&status); + unsigned mesLength = meta->getMessageLength(&status); + unsigned char* streamStart = align(streamBuf, mesAlign); + + // set batch parameters + pb = utl->getXpbBuilder(&status, IXpbBuilder::BATCH, NULL, 0); + // collect per-message statistics + pb->insertInt(&status, IBatch::TAG_RECORD_COUNTS, 1); + + // create batch + const char* sqlStmt1 = "insert into project(proj_id, proj_name) values(?, ?)"; + batch = att->createBatch(&status, tra, 0, sqlStmt1, SAMPLES_DIALECT, meta, + pb->getBufferLength(&status), pb->getBuffer(&status)); + + // fill batch with data record by record + project1->id.set("BAT11"); + project1->name.set("SNGL_REC"); + batch->add(&status, 1, project1.getData()); + + project1->id.set("BAT12"); + project1->name.set("SNGL_REC2"); + batch->add(&status, 1, project1.getData()); + + // execute it + cs = batch->execute(&status, tra); + print_cs(status, cs, utl); + + // fill batch with data using many records at once + stream = streamStart; + + project1->id.set("BAT13"); + project1->name.set("STRM_REC_A"); + putMsg(stream, project1.getData(), mesLength, mesAlign); + + project1->id.set("BAT14"); + project1->name.set("STRM_REC_B"); + putMsg(stream, project1.getData(), mesLength, mesAlign); + + project1->id.set("BAT15"); + project1->name.set("STRM_REC_C"); + putMsg(stream, project1.getData(), mesLength, mesAlign); + + batch->add(&status, 3, streamStart); + + stream = streamStart; + + project1->id.set("BAT15"); // constraint violation + project1->name.set("STRM_REC_D"); + putMsg(stream, project1.getData(), mesLength, mesAlign); + + project1->id.set("BAT16"); + project1->name.set("STRM_REC_E"); + putMsg(stream, project1.getData(), mesLength, mesAlign); + + batch->add(&status, 1, streamStart); + + // execute it + cs = batch->execute(&status, tra); + print_cs(status, cs, utl); + + // close batch + batch->release(); + batch = NULL; + + // + printf("\nPart 2. Simple BLOBs. Multiple errors return.\n"); + // + + // Message to store in a table + FB_MESSAGE(Msg2, ThrowStatusWrapper, + (FB_VARCHAR(5), id) + (FB_VARCHAR(10), name) + (FB_BLOB, desc) + ) project2(&status, master); + project2.clear(); + meta = project2.getMetadata(); + + mesAlign = meta->getAlignment(&status); + mesLength = meta->getMessageLength(&status); + streamStart = align(streamBuf, mesAlign); + + // set batch parameters + pb->clear(&status); + // continue batch processing in case of errors in some messages + pb->insertInt(&status, IBatch::TAG_MULTIERROR, 1); + // enable blobs processing - IDs generated by firebird engine + pb->insertInt(&status, IBatch::TAG_BLOB_POLICY, IBatch::BLOB_ID_ENGINE); + + // create batch + const char* sqlStmt2 = "insert into project(proj_id, proj_name, proj_desc) values(?, ?, ?)"; + batch = att->createBatch(&status, tra, 0, sqlStmt2, SAMPLES_DIALECT, meta, + pb->getBufferLength(&status), pb->getBuffer(&status)); + + // fill batch with data + project2->id.set("BAT21"); + project2->name.set("SNGL_BLOB"); + batch->addBlob(&status, strlen(sqlStmt2), sqlStmt2, &project2->desc, 0, NULL); + batch->appendBlobData(&status, 1, "\n"); + batch->appendBlobData(&status, strlen(sqlStmt1), sqlStmt1); + batch->add(&status, 1, project2.getData()); + + // execute it + cs = batch->execute(&status, tra); + print_cs(status, cs, utl); + + // fill batch with data + project2->id.set("BAT22"); + project2->name.set("SNGL_REC1"); + batch->addBlob(&status, strlen(sqlStmt2), sqlStmt2, &project2->desc, 0, NULL); + batch->add(&status, 1, project2.getData()); + + project2->id.set("BAT22"); + project2->name.set("SNGL_REC2"); // constraint violation + batch->addBlob(&status, 2, "r2", &project2->desc, 0, NULL); + batch->add(&status, 1, project2.getData()); + + project2->id.set("BAT23"); + project2->name.set("SNGL_REC3"); + batch->addBlob(&status, 2, "r3", &project2->desc, 0, NULL); + batch->add(&status, 1, project2.getData()); + + project2->id.set("BAT23"); // constraint violation + project2->name.set("SNGL_REC4"); + batch->addBlob(&status, 2, "r4", &project2->desc, 0, NULL); + batch->add(&status, 1, project2.getData()); + + // execute it + cs = batch->execute(&status, tra); + print_cs(status, cs, utl); + + // close batch + batch->release(); + batch = NULL; + + // + printf("\nPart 3. BLOB stream, including segmented BLOB.\n"); + // + + // use Msg2/project2/sqlStmt2 to store in a table + + // set batch parameters + pb->clear(&status); + // enable blobs processing - blobs are placed in a stream + pb->insertInt(&status, IBatch::TAG_BLOB_POLICY, IBatch::BLOB_STREAM); + + // create batch + batch = att->createBatch(&status, tra, 0, sqlStmt2, SAMPLES_DIALECT, meta, + pb->getBufferLength(&status), pb->getBuffer(&status)); + + unsigned blobAlign = batch->getBlobAlignment(&status); + + // prepare blob IDs + ISC_QUAD v1={0,1}, v2={0,2}, v3={0,3}; + + // send messages to batch + project2->id.set("BAT31"); + project2->name.set("STRM_BLB_A"); + project2->desc = v1; + batch->add(&status, 1, project2.getData()); + + project2->id.set("BAT32"); + project2->name.set("STRM_BLB_B"); + project2->desc = v2; + batch->add(&status, 1, project2.getData()); + + project2->id.set("BAT33"); + project2->name.set("STRM_BLB_C"); + project2->desc = v3; + batch->add(&status, 1, project2.getData()); + + // prepare blobs in the stream buffer + + const char* d1 = "1111111111111111111"; + const char* d2 = "22222222222222222222"; + const char* d3 = "33333333333333333333333333333333333333333333333333333"; + + stream = streamStart; + putBlob(stream, d1, strlen(d1), blobAlign, &v1); + putBlob(stream, d2, strlen(d2), blobAlign, &v2); + putBlob(stream, d3, strlen(d3), blobAlign, &v3); + + batch->addBlobStream(&status, stream - streamStart, streamStart); + + // Put segmented Blob in the stream + + // add message + ISC_QUAD vSeg={0,10}; + project2->id.set("BAT35"); + project2->name.set("STRM_B_SEG"); + project2->desc = vSeg; + batch->add(&status, 1, project2.getData()); + + // build BPB + pb->dispose(); + pb = NULL; + pb = utl->getXpbBuilder(&status, IXpbBuilder::BPB, NULL, 0); + pb->insertInt(&status, isc_bpb_type, isc_bpb_type_segmented); + + // make stream + stream = streamStart; + unsigned* size = putBlobHdr(stream, blobAlign, &vSeg, pb->getBufferLength(&status), pb->getBuffer(&status)); + *size += putSegment(stream, d1); + *size += putSegment(stream, "\n"); + *size += putSegment(stream, d2); + *size += putSegment(stream, "\n"); + *size += putSegment(stream, d3); + + // add stream to the batch + stream = align(stream, blobAlign); + batch->addBlobStream(&status, stream - streamStart, streamStart); + + // execute batch + cs = batch->execute(&status, tra); + print_cs(status, cs, utl); + + // + printf("\nPart 4. BLOB created using IBlob interface.\n"); + // + + // use Msg2/project2/sqlStmt2 to store in a table + // registerBlob() may be called in BLOB_STREAM batch, ID should be generated by user in this case + // also demonstrates execution of same batch multiple times + + // create blob + ISC_QUAD realId; + IBlob* blob = att->createBlob(&status, tra, &realId, 0, NULL); + const char* text = "Blob created using traditional API"; + blob->putSegment(&status, strlen(text), text); + blob->close(&status); + + // add message + project2->id.set("BAT38"); + project2->name.set("FRGN_BLB"); + project2->desc = v1; // after execute may reuse IDs + batch->registerBlob(&status, &realId, &project2->desc); + batch->add(&status, 1, project2.getData()); + + // execute it + cs = batch->execute(&status, tra); + print_cs(status, cs, utl); + + // cleanup + batch->release(); + batch = NULL; + tra->commit(&status); + tra = NULL; + att->detach(&status); + att = NULL; + } + catch (const FbException& error) + { + // handle error + rc = 1; + errPrint(error.getStatus()); + } + + // release interfaces after error caught + if (cs) + cs->dispose(); + if (batch) + batch->release(); + if (tra) + tra->release(); + if (att) + att->release(); + + // cleanup + if (pb) + pb->dispose(); + status.dispose(); + prov->release(); + + return rc; +} diff --git a/src/common/MsgMetadata.cpp b/src/common/MsgMetadata.cpp index 05d0084c3a..bcc02a2e87 100644 --- a/src/common/MsgMetadata.cpp +++ b/src/common/MsgMetadata.cpp @@ -295,21 +295,34 @@ void MsgMetadata::addItem(const MetaName& name, bool nullable, const dsc& desc) // returns ~0 on success or index of not finished item unsigned MsgMetadata::makeOffsets() { - length = 0; + length = alignedLength = 0; + alignment = type_alignments[dtype_short]; // NULL indicator for (unsigned n = 0; n < items.getCount(); ++n) { Item* param = &items[n]; if (!param->finished) { - length = 0; + length = alignment = 0; return n; } + + unsigned dtype; length = fb_utils::sqlTypeToDsc(length, param->type, param->length, - NULL /*dtype*/, NULL /*length*/, ¶m->offset, ¶m->nullInd); + &dtype, NULL /*length*/, ¶m->offset, ¶m->nullInd); + + if (dtype >= DTYPE_TYPE_MAX) + { + length = alignment = 0; + return n; + } + + alignment = MAX(alignment, type_alignments[dtype]); } - return ~0; + alignedLength = FB_ALIGN(length, alignment); + + return ~0u; } diff --git a/src/common/MsgMetadata.h b/src/common/MsgMetadata.h index f7339c325f..f177a11a86 100644 --- a/src/common/MsgMetadata.h +++ b/src/common/MsgMetadata.h @@ -99,20 +99,26 @@ public: public: explicit MsgMetadata(MsgMetadata* from) : items(getPool(), from->items), - length(from->length) + length(from->length), + alignment(from->alignment), + alignedLength(from->alignedLength) { } explicit MsgMetadata(IMessageMetadata* from) : items(getPool()), - length(0) + length(0), + alignment(0), + alignedLength(0) { assign(from); } MsgMetadata() : items(getPool()), - length(0) + length(0), + alignment(0), + alignedLength(0) { } @@ -266,6 +272,16 @@ public: return length; } + unsigned getAlignment(CheckStatusWrapper* /*status*/) + { + return alignment; + } + + unsigned getAlignedLength(CheckStatusWrapper* /*status*/) + { + return alignedLength; + } + public: void addItem(const MetaName& name, bool nullable, const dsc& desc); unsigned makeOffsets(); @@ -281,7 +297,7 @@ private: private: ObjectsArray items; - unsigned length; + unsigned length, alignment, alignedLength; }; //class AttMetadata : public IMessageMetadataBaseImpl diff --git a/src/common/classes/BatchCompletionState.h b/src/common/classes/BatchCompletionState.h new file mode 100644 index 0000000000..3035589830 --- /dev/null +++ b/src/common/classes/BatchCompletionState.h @@ -0,0 +1,185 @@ +/* + * The contents of this file are subject to the Initial + * Developer's Public License Version 1.0 (the "License"); + * you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * http://www.ibphoenix.com/main.nfs?a=ibphoenix&page=ibp_idpl. + * + * Software distributed under the License is distributed AS IS, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. + * See the License for the specific language governing rights + * and limitations under the License. + * + * The Original Code was created by Alexander Peshkov + * for the Firebird Open Source RDBMS project. + * + * Copyright (c) 2017 Alexander Peshkov + * and all contributors signed below. + * + * All Rights Reserved. + * Contributor(s): ________________________________ + */ + +#include "firebird.h" + +#include "../common/classes/auto.h" +#include "../common/classes/array.h" +#include "../common/utils_proto.h" + +namespace Firebird { + + class Transliterate + { + public: + virtual void transliterate(IStatus* status) = 0; + }; + + class BatchCompletionState FB_FINAL : + public DisposeIface > + { + public: + BatchCompletionState(bool storeCounts, ULONG lim) + : rare(getPool()), + reccount(0u), + detailedLimit(lim) + { + if (storeCounts) + array = FB_NEW_POOL(getPool()) DenseArray(getPool()); + } + + ~BatchCompletionState() + { + for (unsigned i = 0; i < rare.getCount() && rare[i].second; ++i) + rare[i].second->dispose(); + } + + void dispose() + { + delete this; + } + + void regError(IStatus* errStatus, Transliterate* transliterate) + { + IStatus* newVector = nullptr; + if (rare.getCount() < detailedLimit) + { + newVector = errStatus->clone(); + if (transliterate) + transliterate->transliterate(newVector); + } + rare.add(StatusPair(reccount, newVector)); + + regUpdate(IBatchCompletionState::EXECUTE_FAILED); + } + + void regErrorAt(ULONG at, IStatus* errStatus) + { + IStatus* newVector = nullptr; + if ((rare.getCount() < detailedLimit) && errStatus) + newVector = errStatus->clone(); + rare.add(StatusPair(at, newVector)); + } + + void regUpdate(SLONG count) + { + if (array) + array->push(count); + + ++reccount; + } + + void regSize(ULONG total) + { + reccount = total; + } + + // IBatchCompletionState implementation + unsigned getSize(CheckStatusWrapper*) + { + return reccount; + } + + int getState(CheckStatusWrapper* status, unsigned pos) + { + try + { + if (pos >= reccount) + (Arg::Gds(isc_random) << "Position is out of range").raise(); + if (array) + return (*array)[pos]; + + ULONG index = find(pos); + return (index >= rare.getCount() || rare[index].first != pos) ? + SUCCESS_NO_INFO : EXECUTE_FAILED; + } + catch (const Exception& ex) + { + ex.stuffException(status); + } + return 0; + } + + unsigned findError(CheckStatusWrapper* status, unsigned pos) + { + try + { + ULONG index = find(pos); + if (index < rare.getCount()) + return rare[index].first; + } + catch (const Exception& ex) + { + ex.stuffException(status); + } + return NO_MORE_ERRORS; + } + + void getStatus(CheckStatusWrapper* status, IStatus* to, unsigned pos) + { + try + { + if (pos >= reccount) + (Arg::Gds(isc_random) << "Position is out of range").raise(); + + ULONG index = find(pos); + if (index < rare.getCount() && rare[index].first == pos) + { + if (rare[index].second) + { + CheckStatusWrapper w(to); + fb_utils::copyStatus(&w, rare[index].second); + return; + } + (Arg::Gds(isc_random) << "Detailed error info is missing in batch").raise(); + } + } + catch (const Exception& ex) + { + ex.stuffException(status); + } + } + + private: + typedef Pair > StatusPair; + typedef Array RarefiedArray; + RarefiedArray rare; + typedef Array DenseArray; + AutoPtr array; + ULONG reccount, detailedLimit; + + ULONG find(ULONG recno) const + { + ULONG high = rare.getCount(), low = 0; + while (high > low) + { + ULONG med = (high + low) / 2; + if (recno > rare[med].first) + low = med + 1; + else + high = med; + } + + return low; + } + }; +} diff --git a/src/common/classes/stack.h b/src/common/classes/stack.h index 275f0dfe03..8ec2bbf52c 100644 --- a/src/common/classes/stack.h +++ b/src/common/classes/stack.h @@ -129,7 +129,7 @@ namespace Firebird { delete stk_cache; } - void push(Object e) + void push(const Object& e) { if (!stk && stk_cache) { @@ -184,7 +184,7 @@ namespace Firebird { class AutoPushPop { public: - AutoPushPop(Stack& s, Object& o) + AutoPushPop(Stack& s, const Object& o) : stack(s) { stack.push(o); diff --git a/src/common/classes/vector.h b/src/common/classes/vector.h index aea47ab432..6caeb011e7 100644 --- a/src/common/classes/vector.h +++ b/src/common/classes/vector.h @@ -36,7 +36,7 @@ namespace Firebird { // Very fast static array of simple types -template +template class Vector { public: @@ -85,6 +85,14 @@ public: return &data[index]; } + T* removeCount(const FB_SIZE_T index, const FB_SIZE_T n) throw() + { + fb_assert(index + n <= count); + memmove(data + index, data + index + n, sizeof(T) * (count - index - n)); + count -= n; + return &data[index]; + } + void shrink(FB_SIZE_T newCount) throw() { fb_assert(newCount <= count); @@ -118,6 +126,20 @@ public: return data[count]; } + void push(const T* items, const FB_SIZE_T itemsCount) + { + fb_assert(count <= FB_MAX_SIZEOF - itemsCount); + fb_assert(count + itemsCount <= Capacity); + memcpy(data + count, items, sizeof(T) * itemsCount); + count += itemsCount; + } + + void append(const T* items, const FB_SIZE_T itemsCount) + { + push(items, itemsCount); + } + + // This method only assigns "pos" if the element is found. // Maybe we should modify it to iterate directy with "pos". bool find(const T& item, FB_SIZE_T& pos) const @@ -134,7 +156,13 @@ public: } protected: - FB_SIZE_T count; + union + { + FB_SIZE_T count; + A align; + }; + // Do not insert data members between align and data: + // alignment of data is ensured by preceding union T data[Capacity]; }; diff --git a/src/common/config/config.cpp b/src/common/config/config.cpp index 81b0e3671a..1d2e228d7a 100644 --- a/src/common/config/config.cpp +++ b/src/common/config/config.cpp @@ -201,7 +201,8 @@ const Config::ConfigEntry Config::entries[MAX_CONFIG_KEY] = {TYPE_INTEGER, "MaxIdentifierCharLength", (ConfigValue) -1}, {TYPE_BOOLEAN, "AllowEncryptedSecurityDatabase", (ConfigValue) false}, {TYPE_INTEGER, "StatementTimeout", (ConfigValue) 0}, - {TYPE_INTEGER, "ConnectionIdleTimeout", (ConfigValue) 0} + {TYPE_INTEGER, "ConnectionIdleTimeout", (ConfigValue) 0}, + {TYPE_INTEGER, "ClientBatchBuffer", (ConfigValue) (128 * 1024)} }; /****************************************************************************** @@ -832,3 +833,9 @@ unsigned int Config::getConnIdleTimeout() const { return get(KEY_CONN_IDLE_TIMEOUT); } + +unsigned int Config::getClientBatchBuffer() const +{ + return get(KEY_CLIENT_BATCH_BUFFER); +} + diff --git a/src/common/config/config.h b/src/common/config/config.h index cbe037adc6..accaa21350 100644 --- a/src/common/config/config.h +++ b/src/common/config/config.h @@ -145,6 +145,7 @@ public: KEY_ENCRYPT_SECURITY_DATABASE, KEY_STMT_TIMEOUT, KEY_CONN_IDLE_TIMEOUT, + KEY_CLIENT_BATCH_BUFFER, MAX_CONFIG_KEY // keep it last }; @@ -359,6 +360,8 @@ public: unsigned int getStatementTimeout() const; // set in minutes unsigned int getConnIdleTimeout() const; + + unsigned int getClientBatchBuffer() const; }; // Implementation of interface to access master configuration file diff --git a/src/common/utils.cpp b/src/common/utils.cpp index 0507af7592..5a2dda5203 100644 --- a/src/common/utils.cpp +++ b/src/common/utils.cpp @@ -52,6 +52,8 @@ #include "../common/StatusArg.h" #include "../common/os/os_utils.h" #include "../dsql/sqlda_pub.h" +#include "../common/classes/ClumpletReader.h" +#include "../common/StatusArg.h" #ifdef WIN_NT #include @@ -1169,7 +1171,7 @@ unsigned int mergeStatus(ISC_STATUS* const dest, unsigned int space, return copied; } -void copyStatus(Firebird::CheckStatusWrapper* to, const Firebird::CheckStatusWrapper* from) throw() +void copyStatus(Firebird::CheckStatusWrapper* to, const Firebird::IStatus* from) throw() { to->init(); @@ -1625,4 +1627,21 @@ const char* dpbItemUpper(const char* s, FB_SIZE_T l, Firebird::string& buf) return buf.c_str(); } +bool isBpbSegmented(unsigned parLength, const unsigned char* par) +{ + if (parLength && !par) + (Firebird::Arg::Gds(isc_random) << "Malformed BPB").raise(); + + Firebird::ClumpletReader bpb(Firebird::ClumpletReader::Tagged, par, parLength); + if (bpb.getBufferTag() != isc_bpb_version1) + (Firebird::Arg::Gds(isc_random) << "Malformed BPB").raise(); + + if (!bpb.find(isc_bpb_type)) + { + return true; + } + int type = bpb.getInt(); + return type & isc_bpb_type_stream ? false : true; +} + } // namespace fb_utils diff --git a/src/common/utils_proto.h b/src/common/utils_proto.h index e92efb5abe..f62a4f7d63 100644 --- a/src/common/utils_proto.h +++ b/src/common/utils_proto.h @@ -136,7 +136,7 @@ namespace fb_utils unsigned int copyStatus(ISC_STATUS* const to, const unsigned int space, const ISC_STATUS* const from, const unsigned int count) throw(); - void copyStatus(Firebird::CheckStatusWrapper* to, const Firebird::CheckStatusWrapper* from) throw(); + void copyStatus(Firebird::CheckStatusWrapper* to, const Firebird::IStatus* from) throw(); unsigned int mergeStatus(ISC_STATUS* const to, unsigned int space, const Firebird::IStatus* from) throw(); void setIStatus(Firebird::CheckStatusWrapper* to, const ISC_STATUS* from) throw(); unsigned int statusLength(const ISC_STATUS* const status) throw(); @@ -206,6 +206,9 @@ namespace fb_utils if (up) name = up; } + + // Frequently used actions with clumplets + bool isBpbSegmented(unsigned parLength, const unsigned char* par); } // namespace fb_utils #endif // INCLUDE_UTILS_PROTO_H diff --git a/src/dsql/DdlNodes.epp b/src/dsql/DdlNodes.epp index 2e5b5d35ac..83cc536d9a 100644 --- a/src/dsql/DdlNodes.epp +++ b/src/dsql/DdlNodes.epp @@ -931,7 +931,7 @@ void DdlNode::executeDdlTrigger(thread_db* tdbb, jrd_tra* transaction, DdlTrigge context.newObjectName = when == DTW_BEFORE ? oldNewObjectName : objectName; } - Stack::AutoPushPop autoContext(attachment->ddlTriggersContext, context); + Stack::AutoPushPop autoContext(attachment->ddlTriggersContext, &context); AutoSavePoint savePoint(tdbb, transaction); EXE_execute_ddl_triggers(tdbb, transaction, when == DTW_BEFORE, action); diff --git a/src/dsql/DsqlBatch.cpp b/src/dsql/DsqlBatch.cpp new file mode 100644 index 0000000000..f5e9c8b268 --- /dev/null +++ b/src/dsql/DsqlBatch.cpp @@ -0,0 +1,988 @@ +/* + * The contents of this file are subject to the Initial + * Developer's Public License Version 1.0 (the "License"); + * you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * http://www.ibphoenix.com/main.nfs?a=ibphoenix&page=ibp_idpl. + * + * Software distributed under the License is distributed AS IS, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. + * See the License for the specific language governing rights + * and limitations under the License. + * + * The Original Code was created by Alexander Peshkov + * for the Firebird Open Source RDBMS project. + * + * Copyright (c) 2017 Alexander Peshkov + * and all contributors signed below. + * + * All Rights Reserved. + * Contributor(s): ________________________________ + */ + +#include "firebird.h" + +#include "../dsql/DsqlBatch.h" + +#include "../jrd/EngineInterface.h" +#include "../jrd/jrd.h" +#include "../jrd/status.h" +#include "../jrd/exe_proto.h" +#include "../dsql/dsql.h" +#include "../dsql/errd_proto.h" +#include "../common/classes/ClumpletReader.h" +#include "../common/classes/auto.h" +#include "../common/classes/fb_string.h" +#include "../common/utils_proto.h" +#include "../common/classes/BatchCompletionState.h" + +using namespace Firebird; +using namespace Jrd; + +namespace { + const char* const TEMP_NAME = "fb_batch"; + const UCHAR initBlobParameters[] = {isc_bpb_version1, isc_bpb_type, 1, isc_bpb_type_stream}; + + class JTransliterate : public Firebird::Transliterate + { + public: + JTransliterate(thread_db* tdbb) + : m_tdbb(tdbb) + { } + + void transliterate(IStatus* status) + { + JRD_transliterate(m_tdbb, status); + } + + private: + thread_db* m_tdbb; + }; +} + +DsqlBatch::DsqlBatch(dsql_req* req, const dsql_msg* /*message*/, IMessageMetadata* inMeta, ClumpletReader& pb) + : m_request(req), + m_batch(NULL), + m_meta(inMeta), + m_messages(m_request->getPool()), + m_blobs(m_request->getPool()), + m_blobMap(m_request->getPool()), + m_blobMeta(m_request->getPool()), + m_defaultBpb(m_request->getPool()), + m_messageSize(0), + m_alignedMessage(0), + m_alignment(0), + m_flags(0), + m_detailed(DETAILED_LIMIT), + m_bufferSize(BUFFER_LIMIT), + m_lastBlob(MAX_ULONG), + m_setBlobSize(false), + m_blobPolicy(IBatch::BLOB_NONE) +{ + memset(&m_genId, 0, sizeof(m_genId)); + + FbLocalStatus st; + m_messageSize = m_meta->getMessageLength(&st); + m_alignedMessage = m_meta->getAlignedLength(&st); + m_alignment = m_meta->getAlignment(&st); + check(&st); + + if (m_messageSize > RAM_BATCH) // hops - message does not fit in our buffer + { + ERRD_post(Arg::Gds(isc_sqlerr) << Arg::Num(-104) << + Arg::Gds(isc_random) << "Message too long"); + } + + for (pb.rewind(); !pb.isEof(); pb.moveNext()) + { + UCHAR t = pb.getClumpTag(); + + switch (t) + { + case IBatch::TAG_MULTIERROR: + case IBatch::TAG_RECORD_COUNTS: + setFlag(t, pb.getInt()); + break; + + case IBatch::TAG_BLOB_POLICY: + m_blobPolicy = pb.getInt(); + + switch (m_blobPolicy) + { + case IBatch::BLOB_ID_ENGINE: + case IBatch::BLOB_ID_USER: + case IBatch::BLOB_STREAM: + break; + default: + m_blobPolicy = IBatch::BLOB_NONE; + break; + } + + break; + + case IBatch::TAG_DETAILED_ERRORS: + m_detailed = pb.getInt(); + if (m_detailed > DETAILED_LIMIT * 4) + m_detailed = DETAILED_LIMIT * 4; + break; + + case IBatch::TAG_BUFFER_BYTES_SIZE: + m_bufferSize = pb.getInt(); + if (m_bufferSize > HARD_BUFFER_LIMIT) + m_bufferSize = HARD_BUFFER_LIMIT; + break; + } + } + + // parse message to detect blobs + unsigned fieldsCount = m_meta->getCount(&st); + check(&st); + + for (unsigned i = 0; i < fieldsCount; ++i) + { + unsigned t = m_meta->getType(&st, i); + check(&st); + + switch (t) + { + case SQL_BLOB: + case SQL_ARRAY: + { + BlobMeta bm; + bm.offset = m_meta->getOffset(&st, i); + check(&st); + bm.nullOffset = m_meta->getNullOffset(&st, i); + check(&st); + m_blobMeta.push(bm); + } + break; + } + } + + // allocate data buffers + m_messages.setBuf(m_bufferSize); + if (m_blobMeta.hasData()) + m_blobs.setBuf(m_bufferSize); + + // assign initial default BPB + setDefBpb(FB_NELEM(initBlobParameters), initBlobParameters); +} + + +DsqlBatch::~DsqlBatch() +{ + if (m_batch) + m_batch->resetHandle(); + if (m_request) + m_request->req_batch = NULL; +} + +Attachment* DsqlBatch::getAttachment() const +{ + return m_request->req_dbb->dbb_attachment; +} + +void DsqlBatch::setInterfacePtr(JBatch* interfacePtr) throw() +{ + fb_assert(!m_batch); + m_batch = interfacePtr; +} + +DsqlBatch* DsqlBatch::open(thread_db* tdbb, dsql_req* req, IMessageMetadata* inMetadata, + unsigned parLength, const UCHAR* par) +{ + SET_TDBB(tdbb); + Jrd::ContextPoolHolder context(tdbb, &req->getPool()); + + // Validate cursor or batch being not already open + + if (req->req_cursor) + { + ERRD_post(Arg::Gds(isc_sqlerr) << Arg::Num(-502) << + Arg::Gds(isc_dsql_cursor_open_err)); + } + + if (req->req_batch) + { + ERRD_post(Arg::Gds(isc_sqlerr) << Arg::Num(-502) << + Arg::Gds(isc_random) << "Request has active batch"); + } + + // Sanity checks before creating batch + + if (!req->req_request) + { + ERRD_post(Arg::Gds(isc_sqlerr) << Arg::Num(-504) << + Arg::Gds(isc_unprepared_stmt)); + } + + const DsqlCompiledStatement* statement = req->getStatement(); + + if (statement->getFlags() & DsqlCompiledStatement::FLAG_ORPHAN) + { + ERRD_post(Arg::Gds(isc_sqlerr) << Arg::Num(-901) << + Arg::Gds(isc_bad_req_handle)); + } + + switch (statement->getType()) + { + case DsqlCompiledStatement::TYPE_INSERT: + case DsqlCompiledStatement::TYPE_DELETE: + case DsqlCompiledStatement::TYPE_UPDATE: + case DsqlCompiledStatement::TYPE_EXEC_PROCEDURE: + case DsqlCompiledStatement::TYPE_EXEC_BLOCK: + break; + + default: + ERRD_post(Arg::Gds(isc_sqlerr) << Arg::Num(-901) << + Arg::Gds(isc_random) << "Invalid type of statement used in batch"); + } + + const dsql_msg* message = statement->getSendMsg(); + if (! (inMetadata && message && req->parseMetadata(inMetadata, message->msg_parameters))) + { + ERRD_post(Arg::Gds(isc_sqlerr) << Arg::Num(-901) << + Arg::Gds(isc_random) << "Statement used in batch must have parameters"); + } + + // Open reader for parameters block + + ClumpletReader pb(ClumpletReader::WideTagged, par, parLength); + if (pb.getBufferLength() && (pb.getBufferTag() != IBatch::VERSION1)) + ERRD_post(Arg::Gds(isc_random) << "Invalid tag in parameters block"); + + // Create batch + + DsqlBatch* b = FB_NEW_POOL(req->getPool()) DsqlBatch(req, message, inMetadata, pb); + req->req_batch = b; + return b; +} + +IMessageMetadata* DsqlBatch::getMetadata(thread_db* tdbb) +{ + m_meta->addRef(); + return m_meta; +} + +void DsqlBatch::add(thread_db* tdbb, ULONG count, const void* inBuffer) +{ + if (!count) + return; + m_messages.align(m_alignment); + m_messages.put(inBuffer, (count - 1) * m_alignedMessage + m_messageSize); +} + +void DsqlBatch::blobCheckMeta() +{ + if (!m_blobMeta.hasData()) + { + ERRD_post(Arg::Gds(isc_sqlerr) << Arg::Num(-104) << + Arg::Gds(isc_random) << "There are no blobs in associated statement"); + } +} + +void DsqlBatch::blobCheckMode(bool stream, const char* fname) +{ + blobCheckMeta(); + + switch (m_blobPolicy) + { + case IBatch::BLOB_ID_ENGINE: + case IBatch::BLOB_ID_USER: + if (!stream) + return; + break; + case IBatch::BLOB_STREAM: + if (stream) + return; + break; + } + + ERRD_post(Arg::Gds(isc_sqlerr) << Arg::Num(-104) << + Arg::Gds(isc_random) << "This *** call can't be used with current blob policy" << + Arg::Gds(isc_random) << fname); +} + +void DsqlBatch::blobSetSize() +{ + // Store size of previous blob if it was changed by appendBlobData() + unsigned blobSize = m_blobs.getSize(); + if (m_setBlobSize) + { + blobSize -= (m_lastBlob + SIZEOF_BLOB_HEAD); + m_blobs.put3(&blobSize, sizeof(blobSize), m_lastBlob + sizeof(ISC_QUAD)); + m_setBlobSize = false; + } +} + +void DsqlBatch::blobPrepare() +{ + blobSetSize(); + + // Align blob stream + m_blobs.align(BLOB_STREAM_ALIGN); +} + +void DsqlBatch::setDefaultBpb(thread_db* tdbb, unsigned parLength, const unsigned char* par) +{ + if (m_blobs.getSize()) + { + ERRD_post(Arg::Gds(isc_sqlerr) << Arg::Num(-104) << + Arg::Gds(isc_random) << "setDefaultBpb() call can be used only with empty batch (no blobs added)"); + } + setDefBpb(parLength, par); +} + +void DsqlBatch::setDefBpb(unsigned parLength, const unsigned char* par) +{ + m_defaultBpb.clear(); + m_defaultBpb.add(par, parLength); + setFlag(FLAG_DEFAULT_SEGMENTED, fb_utils::isBpbSegmented(m_defaultBpb.getCount(), m_defaultBpb.begin())); +} + +void DsqlBatch::addBlob(thread_db* tdbb, ULONG length, const void* inBuffer, ISC_QUAD* blobId, + unsigned parLength, const unsigned char* par) +{ + blobCheckMode(false, "addBlob"); + blobPrepare(); + + // Get ready to appendBlobData() + m_lastBlob = m_blobs.getSize(); + fb_assert(m_lastBlob % BLOB_STREAM_ALIGN == 0); + + // Generate auto blob ID if needed + if (m_blobPolicy == IBatch::BLOB_ID_ENGINE) + genBlobId(blobId); + + // Determine type of current blob + setFlag(FLAG_CURRENT_SEGMENTED, parLength ? fb_utils::isBpbSegmented(parLength, par) : m_flags & (1 << FLAG_DEFAULT_SEGMENTED)); + + // Store header + m_blobs.put(blobId, sizeof(ISC_QUAD)); + ULONG fullLength = length + parLength; + m_blobs.put(&fullLength, sizeof(ULONG)); + m_blobs.put(&parLength, sizeof(ULONG)); + + // Store BPB + if (parLength) + m_blobs.put(par, parLength); + + // Finally store user data + putSegment(length, inBuffer); +} + +void DsqlBatch::appendBlobData(thread_db* tdbb, ULONG length, const void* inBuffer) +{ + blobCheckMode(false, "appendBlobData"); + + if (m_lastBlob == MAX_ULONG) + { + ERRD_post(Arg::Gds(isc_sqlerr) << Arg::Num(-104) << + Arg::Gds(isc_random) << "appendBlobData() is used to append data to last blob " + "but no such blob was added to the batch"); + } + + m_setBlobSize = true; + putSegment(length, inBuffer); +} + +void DsqlBatch::putSegment(ULONG length, const void* inBuffer) +{ + if (m_flags & (1 << FLAG_CURRENT_SEGMENTED)) + { + if (length > MAX_USHORT) + { + ERR_post(Arg::Gds(isc_imp_exc) << Arg::Gds(isc_blobtoobig) << + Arg::Gds(isc_random) << "Segment size >= 64Kb"); + } + USHORT l = length; + m_blobs.align(IBatch::BLOB_SEGHDR_ALIGN); + m_blobs.put(&l, sizeof(l)); + m_setBlobSize = true; + } + m_blobs.put(inBuffer, length); +} + +void DsqlBatch::addBlobStream(thread_db* tdbb, unsigned length, const void* inBuffer) +{ + // Sanity checks + if (length == 0) + return; + if (length % BLOB_STREAM_ALIGN) + { + ERRD_post(Arg::Gds(isc_sqlerr) << Arg::Num(-104) << + Arg::Gds(isc_random) << "Portions of data, passed as blob stream, should have size " + "multiple to the alignment required for blobs"); + } + + blobCheckMode(true, "addBlobStream"); + blobPrepare(); + + // We have no idea where is the last blob located in the stream + m_lastBlob = MAX_ULONG; + + // store stream for further processing + fb_assert(m_blobs.getSize() % BLOB_STREAM_ALIGN == 0); + m_blobs.put(inBuffer, length); +} + +void DsqlBatch::registerBlob(thread_db*, const ISC_QUAD* existingBlob, ISC_QUAD* blobId) +{ + blobCheckMeta(); + + // Generate auto blob ID if needed + if (m_blobPolicy == IBatch::BLOB_ID_ENGINE) + genBlobId(blobId); + + registerBlob(existingBlob, blobId); +} + +void DsqlBatch::registerBlob(const ISC_QUAD* engineBlob, const ISC_QUAD* batchBlob) +{ + ISC_QUAD* idPtr = m_blobMap.put(*batchBlob); + if (!idPtr) + { + ERRD_post(Arg::Gds(isc_sqlerr) << Arg::Num(-104) << + Arg::Gds(isc_random) << "Repeated BlobId in registerBlob(): is ***"); + } + + *idPtr = *engineBlob; +} + +Firebird::IBatchCompletionState* DsqlBatch::execute(thread_db* tdbb) +{ + // todo - add new trace event here + // TraceDSQLExecute trace(req_dbb->dbb_attachment, this); + + jrd_tra* transaction = tdbb->getTransaction(); + + // execution timer + thread_db::TimerGuard timerGuard(tdbb, m_request->setupTimer(tdbb), true); + + // sync internal buffers + if (!m_messages.done()) + { + ERRD_post(Arg::Gds(isc_sqlerr) << Arg::Num(-104) << + Arg::Gds(isc_random) << "Internal message buffer overflow - batch too big"); + } + + // insert blobs here + if (m_blobMeta.hasData()) + { + // This code expects the following to work correctly + fb_assert(RAM_BATCH % BLOB_STREAM_ALIGN == 0); + + blobSetSize(); // Needed after appendBlobData() + + if (!m_blobs.done()) + { + ERRD_post(Arg::Gds(isc_sqlerr) << Arg::Num(-104) << + Arg::Gds(isc_random) << "Internal BLOB buffer overflow - batch too big"); + } + + struct BlobFlow + { + ULONG remains; + UCHAR* data; + ULONG currentBlobSize; + ULONG byteCount; + + BlobFlow() + : remains(0), data(NULL), currentBlobSize(0), byteCount(0) + { } + + void newHdr(ULONG blobSize) + { + currentBlobSize = blobSize; + move3(SIZEOF_BLOB_HEAD); + } + + void move(ULONG step) + { + move3(step); + currentBlobSize -= step; + } + + bool align(ULONG alignment) + { + ULONG a = byteCount % alignment; + if (a) + { + a = alignment - a; + move3(a); + if (currentBlobSize) + currentBlobSize -= a; + } + return a; + } + +private: + void move3(ULONG step) + { + data += step; + byteCount += step; + remains -= step; + } + }; + BlobFlow flow; + blb* blob = nullptr; + try + { + while ((flow.remains = m_blobs.get(&flow.data)) > 0) + { + while (flow.remains) + { + // should we get next blob header + if (!flow.currentBlobSize) + { + // align data stream + if (flow.align(BLOB_STREAM_ALIGN)) + continue; + + // check for partial header in the buffer + if (flow.remains < SIZEOF_BLOB_HEAD) + flow.remains = m_blobs.reget(flow.remains, &flow.data, BLOB_STREAM_ALIGN); + if (flow.remains < SIZEOF_BLOB_HEAD) + { + ERRD_post(Arg::Gds(isc_sqlerr) << Arg::Num(-104) << + Arg::Gds(isc_random) << "Blob buffer format error: useless data remained in buffer"); + } + + // parse blob header + fb_assert(intptr_t(flow.data) % BLOB_STREAM_ALIGN == 0); + ISC_QUAD* batchBlobId = reinterpret_cast(flow.data); + ULONG* blobSize = reinterpret_cast(flow.data + sizeof(ISC_QUAD)); + ULONG* bpbSize = reinterpret_cast(flow.data + sizeof(ISC_QUAD) + sizeof(ULONG)); + flow.newHdr(*blobSize); + ULONG currentBpbSize = *bpbSize; + + if (batchBlobId->gds_quad_high == 0 && batchBlobId->gds_quad_low == 0) + { + // Sanity check + if (*bpbSize) + { + ERRD_post(Arg::Gds(isc_sqlerr) << Arg::Num(-104) << + Arg::Gds(isc_random) << "Blob buffer format error: blob continuation should not contain BPB"); + } + } + else + { + // get BPB + Bpb localBpb; + Bpb* bpb; + bool segmentedMode; + if (currentBpbSize) + { + if (currentBpbSize > flow.remains) + flow.remains = m_blobs.reget(flow.remains, &flow.data, BLOB_STREAM_ALIGN); + if (currentBpbSize > flow.remains) + { + ERRD_post(Arg::Gds(isc_sqlerr) << Arg::Num(-104) << + Arg::Gds(isc_random) << "Blob buffer format error: size of BPB greater than remaining data"); // <BLB_close(tdbb); + blob = nullptr; + } + bid engineBlobId; + blob = blb::create2(tdbb, transaction, &engineBlobId, bpb->getCount(), + bpb->begin(), true); + registerBlob(reinterpret_cast(&engineBlobId), batchBlobId); + } + } + + // store data + ULONG dataSize = MIN(flow.currentBlobSize, flow.remains); + if (dataSize) + { + if (m_flags & (1 << FLAG_CURRENT_SEGMENTED)) + { + if (flow.align(IBatch::BLOB_SEGHDR_ALIGN)) + continue; + + fb_assert(dataSize >= sizeof(USHORT)); + USHORT* segSize = reinterpret_cast(flow.data); + flow.move(sizeof(USHORT)); + + dataSize = *segSize; + if (dataSize > flow.currentBlobSize) + { + ERRD_post(Arg::Gds(isc_sqlerr) << Arg::Num(-104) << + Arg::Gds(isc_random) << "Blob buffer format error: size of segment exceeds remaining data"); // < flow.remains) + { + flow.remains = m_blobs.reget(flow.remains, &flow.data, BLOB_STREAM_ALIGN); + if (dataSize > flow.remains) + { + ERRD_post(Arg::Gds(isc_sqlerr) << Arg::Num(-104) << + Arg::Gds(isc_random) << "Blob buffer format error: size of segment exceeds RAM buffer"); // <BLB_put_segment(tdbb, flow.data, dataSize); + flow.move(dataSize); + } + } + + m_blobs.remained(0); + } + + if (blob) + { + blob->BLB_close(tdbb); + blob = nullptr; + } + } + catch (const Exception&) + { + if (blob) + blob->BLB_cancel(tdbb); + cancel(tdbb); + + throw; + } + } + + // execute request + m_request->req_transaction = transaction; + jrd_req* req = m_request->req_request; + fb_assert(req); + + // prepare completion interface + AutoPtr > completionState + (FB_NEW BatchCompletionState(m_flags & (1 << IBatch::TAG_RECORD_COUNTS), m_detailed)); + AutoSetRestore batchFlag(&req->req_batch, true); + const dsql_msg* message = m_request->getStatement()->getSendMsg(); + bool startRequest = true; + + // process messages + ULONG remains; + UCHAR* data; + while ((remains = m_messages.get(&data)) > 0) + { + if (remains < m_messageSize) + { + ERRD_post(Arg::Gds(isc_sqlerr) << Arg::Num(-104) << + Arg::Gds(isc_random) << "Internal error: useless data remained in batch buffer"); + } + + while (remains >= m_messageSize) + { + if (startRequest) + { + EXE_unwind(tdbb, req); + EXE_start(tdbb, req, transaction); + startRequest = false; + } + + // skip alignment data + UCHAR* alignedData = FB_ALIGN(data, m_alignment); + if (alignedData != data) + { + remains -= (alignedData - data); + data = alignedData; + continue; + } + + // translate blob IDs + fb_assert(intptr_t(data) % m_alignment == 0); + for (unsigned i = 0; i < m_blobMeta.getCount(); ++i) + { + const SSHORT* nullFlag = reinterpret_cast(&data[m_blobMeta[i].nullOffset]); + if (*nullFlag) + continue; + + ISC_QUAD* id = reinterpret_cast(&data[m_blobMeta[i].offset]); + ISC_QUAD newId; + if (!m_blobMap.get(*id, newId)) + { + ERRD_post(Arg::Gds(isc_sqlerr) << Arg::Num(-104) << + Arg::Gds(isc_random) << "Unknown blob ID in the message: is ***" << + Arg::Gds(isc_random) << Arg::Num(id->gds_quad_high) << + Arg::Gds(isc_random) << Arg::Num(id->gds_quad_low)); + } + + m_blobMap.remove(*id); + *id = newId; + } + + // map message to internal engine format + m_request->mapInOut(tdbb, false, message, m_meta, NULL, data); + data += m_messageSize; + remains -= m_messageSize; + + UCHAR* msgBuffer = m_request->req_msg_buffers[message->msg_buffer_number]; + DEB_BATCH(fprintf(stderr, "\n\n+++ Send\n\n")); + try + { + ULONG before = req->req_records_inserted + req->req_records_updated + + req->req_records_deleted; + EXE_send(tdbb, req, message->msg_number, message->msg_length, msgBuffer); + ULONG after = req->req_records_inserted + req->req_records_updated + + req->req_records_deleted; + completionState->regUpdate(after - before); + } + catch (const Exception& ex) + { + FbLocalStatus status; + ex.stuffException(&status); + tdbb->tdbb_status_vector->init(); + + JTransliterate trLit(tdbb); + completionState->regError(&status, &trLit); + + if (!(m_flags & (1 << IBatch::TAG_MULTIERROR))) + { + cancel(tdbb); + remains = 0; + break; + } + + startRequest = true; + } + } + + UCHAR* alignedData = FB_ALIGN(data, m_alignment); + m_messages.remained(remains, alignedData - data); + } + + // reset to initial state + cancel(tdbb); + + return completionState.release(); +} + +void DsqlBatch::cancel(thread_db* tdbb) +{ + m_messages.clear(); + if (m_blobMeta.hasData()) + { + m_blobs.clear(); + m_setBlobSize = false; + m_lastBlob = MAX_ULONG; + memset(&m_genId, 0, sizeof(m_genId)); + m_blobMap.clear(); + } +} + +void DsqlBatch::genBlobId(ISC_QUAD* blobId) +{ + if (++m_genId.gds_quad_low == 0) + ++m_genId.gds_quad_high; + memcpy(blobId, &m_genId, sizeof(m_genId)); +} + +void DsqlBatch::DataCache::setBuf(ULONG size) +{ + m_limit = size; + + // create ram cache + fb_assert(!m_cache); + m_cache = FB_NEW_POOL(getPool()) Cache; +} + +void DsqlBatch::DataCache::put3(const void* data, ULONG dataSize, ULONG offset) +{ + // This assertion guarantees that data always fits as a whole into m_cache or m_space, + // never placed half in one storage, half - in another. + fb_assert((DsqlBatch::RAM_BATCH % dataSize == 0) && (offset % dataSize == 0)); + + if (offset >= m_used) + { + // data in cache + UCHAR* to = m_cache->begin(); + to += (offset - m_used); + fb_assert(to < m_cache->end()); + memcpy(to, data, dataSize); + } + else + { + const FB_UINT64 writtenBytes = m_space->write(offset, data, dataSize); + fb_assert(writtenBytes == dataSize); + } +} + +void DsqlBatch::DataCache::put(const void* d, ULONG dataSize) +{ + if (m_used + (m_cache ? m_cache->getCount() : 0) + dataSize > m_limit) + { + ERRD_post(Arg::Gds(isc_sqlerr) << Arg::Num(-104) << + Arg::Gds(isc_random) << "Internal buffer overflow - batch too big"); + } + + const UCHAR* data = reinterpret_cast(d); + + // Coefficient affecting direct data write to tempspace + const ULONG K = 4; + + // ensure ram cache presence + fb_assert(m_cache); + + // swap to secondary cache if needed + if (m_cache->getCount() + dataSize > m_cache->getCapacity()) + { + // store data in the end of ram cache if needed + // avoid copy in case of huge buffer passed + ULONG delta = m_cache->getCapacity() - m_cache->getCount(); + if (dataSize - delta < m_cache->getCapacity() / K) + { + m_cache->append(data, delta); + data += delta; + dataSize -= delta; + } + + // swap ram cache to tempspace + if (!m_space) + m_space = FB_NEW_POOL(getPool()) TempSpace(getPool(), TEMP_NAME); + + const FB_UINT64 writtenBytes = m_space->write(m_used, m_cache->begin(), m_cache->getCount()); + fb_assert(writtenBytes == m_cache->getCount()); + m_used += m_cache->getCount(); + m_cache->clear(); + + // in a case of huge buffer write directly to tempspace + if (dataSize > m_cache->getCapacity() / K) + { + const FB_UINT64 writtenBytes = m_space->write(m_used, data, dataSize); + fb_assert(writtenBytes == dataSize); + m_used += dataSize; + return; + } + } + + m_cache->append(data, dataSize); +} + +void DsqlBatch::DataCache::align(ULONG alignment) +{ + ULONG a = getSize() % alignment; + if (a) + { + fb_assert(alignment <= sizeof(SINT64)); + SINT64 zero = 0; + put(&zero, alignment - a); + } +} + +bool DsqlBatch::DataCache::done() +{ + fb_assert(m_cache); + + if (m_cache->getCount() == 0 && m_used == 0) + return true; // false? + + if (m_cache->getCount() && m_used) + { + fb_assert(m_space); + + const FB_UINT64 writtenBytes = m_space->write(m_used, m_cache->begin(), m_cache->getCount()); + fb_assert(writtenBytes == m_cache->getCount()); + m_used += m_cache->getCount(); + m_cache->clear(); + } + + return true; +} + +ULONG DsqlBatch::DataCache::get(UCHAR** buffer) +{ + if (m_used > m_got) + { + // get data from tempspace + ULONG dlen = m_cache->getCount(); + ULONG delta = m_cache->getCapacity() - dlen; + if (delta > m_used - m_got) + delta = m_used - m_got; + UCHAR* buf = m_cache->getBuffer(dlen + delta); + buf += dlen; + const FB_UINT64 readBytes = m_space->read(m_got, buf, delta); + fb_assert(readBytes == delta); + m_got += delta; + } + + if (m_cache->getCount()) + { + if (m_shift) + m_cache->removeCount(0, m_shift); + + // return buffer full of data + *buffer = m_cache->begin(); + fb_assert(intptr_t(*buffer) % FB_ALIGNMENT == 0); + return m_cache->getCount(); + } + + // no more data + *buffer = nullptr; + return 0; +} + +ULONG DsqlBatch::DataCache::reget(ULONG remains, UCHAR** buffer, ULONG alignment) +{ + ULONG a = remains % alignment; + if (a) + { + a = alignment - a; + remains += a; + } + fb_assert(remains < m_cache->getCount()); + + m_cache->removeCount(0, m_cache->getCount() - remains); + ULONG size = get(buffer); + size -= a; + *buffer += a; + return size; +} + +void DsqlBatch::DataCache::remained(ULONG size, ULONG alignment) +{ + if (size > alignment) + { + size -= alignment; + alignment = 0; + } + else + { + alignment -= size; + size = 0; + } + + if (!size) + m_cache->clear(); + else + m_cache->removeCount(0, m_cache->getCount() - size); + + m_shift = alignment; +} + +ULONG DsqlBatch::DataCache::getSize() const +{ + if(!m_cache) + return 0; + + fb_assert((MAX_ULONG - 1) - m_used > m_cache->getCount()); + return m_used + m_cache->getCount(); +} + +void DsqlBatch::DataCache::clear() +{ + m_cache->clear(); + if (m_space && m_used) + m_space->releaseSpace(0, m_used); + m_used = m_got = m_shift = 0; +} diff --git a/src/dsql/DsqlBatch.h b/src/dsql/DsqlBatch.h new file mode 100644 index 0000000000..79779cc02c --- /dev/null +++ b/src/dsql/DsqlBatch.h @@ -0,0 +1,160 @@ +/* + * The contents of this file are subject to the Initial + * Developer's Public License Version 1.0 (the "License"); + * you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * http://www.ibphoenix.com/main.nfs?a=ibphoenix&page=ibp_idpl. + * + * Software distributed under the License is distributed AS IS, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. + * See the License for the specific language governing rights + * and limitations under the License. + * + * The Original Code was created by Alexander Peshkov + * for the Firebird Open Source RDBMS project. + * + * Copyright (c) 2017 Alexander Peshkov + * and all contributors signed below. + * + * All Rights Reserved. + * Contributor(s): ________________________________ + */ + +#ifndef DSQL_BATCH_H +#define DSQL_BATCH_H + +#include "../jrd/TempSpace.h" +#include "../common/classes/alloc.h" +#include "../common/classes/RefCounted.h" +#include "../common/classes/vector.h" +#include "../common/classes/GenericMap.h" + +#define DEB_BATCH(x) + + +namespace Firebird { + +class ClumpletReader; + +} + + +namespace Jrd { + +class dsql_req; +class dsql_msg; +class thread_db; +class JBatch; +class Attachment; + +class DsqlBatch +{ +public: + DsqlBatch(dsql_req* req, const dsql_msg* message, Firebird::IMessageMetadata* inMetadata, + Firebird::ClumpletReader& pb); + ~DsqlBatch(); + + static const ULONG RAM_BATCH = 128 * 1024; + static const ULONG BUFFER_LIMIT = 16 * 1024 * 1024; + static const ULONG HARD_BUFFER_LIMIT = 256 * 1024 * 1024; + static const ULONG DETAILED_LIMIT = 64; + static const ULONG SIZEOF_BLOB_HEAD = sizeof(ISC_QUAD) + 2 * sizeof(ULONG); + static const unsigned BLOB_STREAM_ALIGN = 4; + + static DsqlBatch* open(thread_db* tdbb, dsql_req* req, Firebird::IMessageMetadata* inMetadata, + unsigned parLength, const UCHAR* par); + + Attachment* getAttachment() const; + void setInterfacePtr(JBatch* interfacePtr) throw(); + + void add(thread_db* tdbb, ULONG count, const void* inBuffer); + void addBlob(thread_db* tdbb, ULONG length, const void* inBuffer, ISC_QUAD* blobId, unsigned parLength, const unsigned char* par); + void appendBlobData(thread_db* tdbb, ULONG length, const void* inBuffer); + void addBlobStream(thread_db* tdbb, unsigned length, const void* inBuffer); + void registerBlob(thread_db* tdbb, const ISC_QUAD* existingBlob, ISC_QUAD* blobId); + Firebird::IBatchCompletionState* execute(thread_db* tdbb); + Firebird::IMessageMetadata* getMetadata(thread_db* tdbb); + void cancel(thread_db* tdbb); + void setDefaultBpb(thread_db* tdbb, unsigned parLength, const unsigned char* par); + + // Additional flags - start from the maximum one + static const UCHAR FLAG_DEFAULT_SEGMENTED = 31; + static const UCHAR FLAG_CURRENT_SEGMENTED = 30; + +private: + void genBlobId(ISC_QUAD* blobId); + void blobPrepare(); + void blobSetSize(); + void blobCheckMode(bool stream, const char* fname); + void blobCheckMeta(); + void registerBlob(const ISC_QUAD* engineBlob, const ISC_QUAD* batchBlob); + void setDefBpb(unsigned parLength, const unsigned char* par); + void putSegment(ULONG length, const void* inBuffer); + + void setFlag(UCHAR bit, bool value) + { + if (value) + m_flags |= (1 << bit); + else + m_flags &= ~(1 << bit); + } + + dsql_req* const m_request; + JBatch* m_batch; + Firebird::RefPtr m_meta; + + class DataCache : public Firebird::PermanentStorage + { + public: + DataCache(MemoryPool& p) + : PermanentStorage(p), + m_used(0), m_got(0), m_limit(0), m_shift(0) + { } + + void setBuf(ULONG size); + + void put(const void* data, ULONG dataSize); + void put3(const void* data, ULONG dataSize, ULONG offset); + void align(ULONG alignment); + bool done(); + ULONG get(UCHAR** buffer); + ULONG reget(ULONG size, UCHAR** buffer, ULONG alignment); + void remained(ULONG size, ULONG alignment = 0); + ULONG getSize() const; + void clear(); + + private: + typedef Firebird::Vector Cache; + Firebird::AutoPtr m_cache; + Firebird::AutoPtr m_space; + ULONG m_used, m_got, m_limit, m_shift; + }; + + struct BlobMeta + { + unsigned nullOffset, offset; + }; + + class QuadComparator + { + public: + static bool greaterThan(const ISC_QUAD& i1, const ISC_QUAD& i2) + { + return memcmp(&i1, &i2, sizeof(ISC_QUAD)) > 0; + } + }; + + DataCache m_messages, m_blobs; + Firebird::GenericMap >, QuadComparator> m_blobMap; + Firebird::HalfStaticArray m_blobMeta; + typedef Firebird::HalfStaticArray Bpb; + Bpb m_defaultBpb; + ISC_QUAD m_genId; + ULONG m_messageSize, m_alignedMessage, m_alignment, m_flags, m_detailed, m_bufferSize, m_lastBlob; + bool m_setBlobSize; + UCHAR m_blobPolicy; +}; + +} // namespace + +#endif // DSQL_BATCH_H diff --git a/src/dsql/StmtNodes.cpp b/src/dsql/StmtNodes.cpp index 328fad7e4f..6bc840ddac 100644 --- a/src/dsql/StmtNodes.cpp +++ b/src/dsql/StmtNodes.cpp @@ -6470,14 +6470,16 @@ const StmtNode* PostEventNode::execute(thread_db* tdbb, jrd_req* request, ExeSta static RegisterNode regReceiveNode(blr_receive); +static RegisterNode regReceiveNodeBatch(blr_receive_batch); -DmlNode* ReceiveNode::parse(thread_db* tdbb, MemoryPool& pool, CompilerScratch* csb, const UCHAR /*blrOp*/) +DmlNode* ReceiveNode::parse(thread_db* tdbb, MemoryPool& pool, CompilerScratch* csb, const UCHAR blrOp) { ReceiveNode* node = FB_NEW_POOL(pool) ReceiveNode(pool); USHORT n = csb->csb_blr_reader.getByte(); node->message = csb->csb_rpt[n].csb_message; node->statement = PAR_parse_stmt(tdbb, csb); + node->batchFlag = (blrOp == blr_receive_batch); return node; } @@ -6493,6 +6495,7 @@ string ReceiveNode::internalPrint(NodePrinter& printer) const NODE_PRINT(printer, statement); NODE_PRINT(printer, message); + NODE_PRINT(printer, batchFlag); return "ReceiveNode"; } @@ -6522,6 +6525,11 @@ const StmtNode* ReceiveNode::execute(thread_db* /*tdbb*/, jrd_req* request, ExeS { switch (request->req_operation) { + case jrd_req::req_return: + if (!(request->req_batch && batchFlag)) + break; + // fall into + case jrd_req::req_evaluate: request->req_operation = jrd_req::req_receive; request->req_message = message; @@ -6533,8 +6541,10 @@ const StmtNode* ReceiveNode::execute(thread_db* /*tdbb*/, jrd_req* request, ExeS return statement; default: - return parentStmt; + break; } + + return parentStmt; } diff --git a/src/dsql/StmtNodes.h b/src/dsql/StmtNodes.h index 398eba0b0f..aa9d3c340d 100644 --- a/src/dsql/StmtNodes.h +++ b/src/dsql/StmtNodes.h @@ -1223,7 +1223,8 @@ public: explicit ReceiveNode(MemoryPool& pool) : TypedNode(pool), statement(NULL), - message(NULL) + message(NULL), + batchFlag(false) { } @@ -1240,6 +1241,7 @@ public: public: NestConst statement; NestConst message; + bool batchFlag; }; diff --git a/src/dsql/dsql.cpp b/src/dsql/dsql.cpp index 21c8de18a4..b38460b21a 100644 --- a/src/dsql/dsql.cpp +++ b/src/dsql/dsql.cpp @@ -69,6 +69,7 @@ #include "../common/classes/init.h" #include "../common/utils_proto.h" #include "../common/StatusArg.h" +#include "../dsql/DsqlBatch.h" #ifdef HAVE_CTYPE_H #include @@ -80,9 +81,6 @@ using namespace Firebird; static ULONG get_request_info(thread_db*, dsql_req*, ULONG, UCHAR*); static dsql_dbb* init(Jrd::thread_db*, Jrd::Attachment*); -static void map_in_out(Jrd::thread_db*, dsql_req*, bool, const dsql_msg*, IMessageMetadata*, UCHAR*, - const UCHAR* = NULL); -static USHORT parse_metadata(dsql_req*, IMessageMetadata*, const Array&); static dsql_req* prepareRequest(thread_db*, dsql_dbb*, jrd_tra*, ULONG, const TEXT*, USHORT, bool); static dsql_req* prepareStatement(thread_db*, dsql_dbb*, jrd_tra*, ULONG, const TEXT*, USHORT, bool); static UCHAR* put_item(UCHAR, const USHORT, const UCHAR*, UCHAR*, const UCHAR* const); @@ -212,7 +210,7 @@ DsqlCursor* DSQL_open(thread_db* tdbb, if (!reqTypeWithCursor(statement->getType())) Arg::Gds(isc_no_cursor).raise(); - // Validate cursor being not already open + // Validate cursor or batch being not already open if (request->req_cursor) { @@ -220,6 +218,12 @@ DsqlCursor* DSQL_open(thread_db* tdbb, Arg::Gds(isc_dsql_cursor_open_err)); } + if (request->req_batch) + { + ERRD_post(Arg::Gds(isc_sqlerr) << Arg::Num(-502) << + Arg::Gds(isc_random) << "Request has active batch"); + } + request->req_transaction = *tra_handle; request->execute(tdbb, tra_handle, in_meta, in_msg, out_meta, NULL, false); @@ -297,7 +301,7 @@ bool DsqlDmlRequest::fetch(thread_db* tdbb, UCHAR* msgBuffer) return false; } - map_in_out(tdbb, this, true, message, delayedFormat, msgBuffer); + mapInOut(tdbb, true, message, delayedFormat, msgBuffer); delayedFormat = NULL; trace.fetch(false, ITracePlugin::RESULT_SUCCESS); @@ -681,9 +685,9 @@ void DsqlDmlRequest::execute(thread_db* tdbb, jrd_tra** traHandle, const dsql_msg* message = statement->getSendMsg(); if (message) - map_in_out(tdbb, this, false, message, inMetadata, NULL, inMsg); + mapInOut(tdbb, false, message, inMetadata, NULL, inMsg); - // we need to map_in_out before tracing of execution start to let trace + // we need to mapInOut() before tracing of execution start to let trace // manager know statement parameters values TraceDSQLExecute trace(req_dbb->dbb_attachment, this); @@ -721,7 +725,7 @@ void DsqlDmlRequest::execute(thread_db* tdbb, jrd_tra** traHandle, } if (outMetadata && message) - parse_metadata(this, outMetadata, message->msg_parameters); + parseMetadata(outMetadata, message->msg_parameters); if ((outMsg && message) || isBlock) { @@ -744,7 +748,7 @@ void DsqlDmlRequest::execute(thread_db* tdbb, jrd_tra** traHandle, JRD_receive(tdbb, req_request, message->msg_number, message->msg_length, msgBuffer); if (outMsg) - map_in_out(tdbb, this, true, message, NULL, outMsg); + mapInOut(tdbb, true, message, NULL, outMsg); // if this is a singleton select, make sure there's in fact one record @@ -1015,7 +1019,7 @@ static dsql_dbb* init(thread_db* tdbb, Jrd::Attachment* attachment) /** - map_in_out + mapInOut @brief Map data from external world into message or from message to external world. @@ -1029,10 +1033,10 @@ static dsql_dbb* init(thread_db* tdbb, Jrd::Attachment* attachment) @param in_dsql_msg_buf **/ -static void map_in_out(thread_db* tdbb, dsql_req* request, bool toExternal, const dsql_msg* message, +void dsql_req::mapInOut(thread_db* tdbb, bool toExternal, const dsql_msg* message, IMessageMetadata* meta, UCHAR* dsql_msg_buf, const UCHAR* in_dsql_msg_buf) { - USHORT count = parse_metadata(request, meta, message->msg_parameters); + USHORT count = parseMetadata(meta, message->msg_parameters); // Sanity check @@ -1067,7 +1071,7 @@ static void map_in_out(thread_db* tdbb, dsql_req* request, bool toExternal, cons // Make sure the message given to us is long enough dsc desc; - if (!request->req_user_descs.get(parameter, desc)) + if (!req_user_descs.get(parameter, desc)) desc.clear(); /*** @@ -1085,14 +1089,14 @@ static void map_in_out(thread_db* tdbb, dsql_req* request, bool toExternal, cons Arg::Gds(isc_dsql_sqlvar_index) << Arg::Num(parameter->par_index-1)); } - UCHAR* msgBuffer = request->req_msg_buffers[parameter->par_message->msg_buffer_number]; + UCHAR* msgBuffer = req_msg_buffers[parameter->par_message->msg_buffer_number]; SSHORT* flag = NULL; dsql_par* const null_ind = parameter->par_null; if (null_ind != NULL) { dsc userNullDesc; - if (!request->req_user_descs.get(null_ind, userNullDesc)) + if (!req_user_descs.get(null_ind, userNullDesc)) userNullDesc.clear(); const ULONG null_offset = (IPTR) userNullDesc.dsc_address; @@ -1155,7 +1159,7 @@ static void map_in_out(thread_db* tdbb, dsql_req* request, bool toExternal, cons Arg::Gds(isc_dsql_wrong_param_num) << Arg::Num(count) <getStatement(); + const DsqlCompiledStatement* statement = getStatement(); const dsql_par* parameter; const dsql_par* dbkey; @@ -1164,7 +1168,7 @@ static void map_in_out(thread_db* tdbb, dsql_req* request, bool toExternal, cons { UCHAR* parentMsgBuffer = statement->getParentRequest() ? statement->getParentRequest()->req_msg_buffers[dbkey->par_message->msg_buffer_number] : NULL; - UCHAR* msgBuffer = request->req_msg_buffers[parameter->par_message->msg_buffer_number]; + UCHAR* msgBuffer = req_msg_buffers[parameter->par_message->msg_buffer_number]; fb_assert(parentMsgBuffer); @@ -1194,7 +1198,7 @@ static void map_in_out(thread_db* tdbb, dsql_req* request, bool toExternal, cons UCHAR* parentMsgBuffer = statement->getParentRequest() ? statement->getParentRequest()->req_msg_buffers[rec_version->par_message->msg_buffer_number] : NULL; - UCHAR* msgBuffer = request->req_msg_buffers[parameter->par_message->msg_buffer_number]; + UCHAR* msgBuffer = req_msg_buffers[parameter->par_message->msg_buffer_number]; fb_assert(parentMsgBuffer); @@ -1221,7 +1225,7 @@ static void map_in_out(thread_db* tdbb, dsql_req* request, bool toExternal, cons /** - parse_metadata + parseMetadata @brief Parse the message of a request. @@ -1231,8 +1235,7 @@ static void map_in_out(thread_db* tdbb, dsql_req* request, bool toExternal, cons @param parameters_list **/ -static USHORT parse_metadata(dsql_req* request, IMessageMetadata* meta, - const Array& parameters_list) +USHORT dsql_req::parseMetadata(IMessageMetadata* meta, const Array& parameters_list) { HalfStaticArray parameters; @@ -1306,7 +1309,7 @@ static USHORT parse_metadata(dsql_req* request, IMessageMetadata* meta, if (desc.isText() && desc.getTextType() == ttype_dynamic) desc.setTextType(ttype_none); - request->req_user_descs.put(parameter, desc); + req_user_descs.put(parameter, desc); dsql_par* null = parameter->par_null; if (null) @@ -1317,7 +1320,7 @@ static USHORT parse_metadata(dsql_req* request, IMessageMetadata* meta, desc.dsc_length = sizeof(SSHORT); desc.dsc_address = (UCHAR*)(IPTR) nullOffset; - request->req_user_descs.put(null, desc); + req_user_descs.put(null, desc); } } @@ -1564,6 +1567,7 @@ dsql_req::dsql_req(MemoryPool& pool) req_msg_buffers(req_pool), req_cursor_name(req_pool), req_cursor(NULL), + req_batch(NULL), req_user_descs(req_pool), req_traced(false), req_timeout(0) @@ -1618,10 +1622,10 @@ void dsql_req::setTimeout(unsigned int timeOut) req_timeout = timeOut; } -void dsql_req::setupTimer(thread_db* tdbb) +TimeoutTimer* dsql_req::setupTimer(thread_db* tdbb) { if (statement->getFlags() & JrdStatement::FLAG_INTERNAL) - return; + return req_timer; if (req_request) { @@ -1632,7 +1636,7 @@ void dsql_req::setupTimer(thread_db* tdbb) { if (req_timer) req_timer->setup(0, 0); - return; + return req_timer; } } @@ -1672,6 +1676,8 @@ void dsql_req::setupTimer(thread_db* tdbb) req_timer->setup(timeOut, toutErr); req_timer->start(); } + + return req_timer; } // Release a dynamic request. @@ -1708,6 +1714,12 @@ void dsql_req::destroy(thread_db* tdbb, dsql_req* request, bool drop) if (request->req_cursor) DsqlCursor::close(tdbb, request->req_cursor); + if (request->req_batch) + { + delete request->req_batch; + request->req_batch = nullptr; + } + Jrd::Attachment* att = request->req_dbb->dbb_attachment; const bool need_trace_free = request->req_traced && TraceManager::need_dsql_free(att); if (need_trace_free) @@ -1959,6 +1971,14 @@ static void sql_info(thread_db* tdbb, return; break; + case isc_info_sql_stmt_blob_align: + value = DsqlBatch::BLOB_STREAM_ALIGN; + + length = put_vax_long(buffer, value); + if (!(info = put_item(item, length, buffer, info, end_info))) + return; + break; + case isc_info_sql_get_plan: case isc_info_sql_explain_plan: { diff --git a/src/dsql/dsql.h b/src/dsql/dsql.h index 774a5017b9..654daa0e6a 100644 --- a/src/dsql/dsql.h +++ b/src/dsql/dsql.h @@ -598,7 +598,11 @@ public: // Evaluate actual timeout value, consider config- and session-level timeout values, // setup and start timer - void setupTimer(thread_db* tdbb); + TimeoutTimer* setupTimer(thread_db* tdbb); + + USHORT parseMetadata(Firebird::IMessageMetadata* meta, const Firebird::Array& parameters_list); + void mapInOut(Jrd::thread_db* tdbb, bool toExternal, const dsql_msg* message, Firebird::IMessageMetadata* meta, + UCHAR* dsql_msg_buf, const UCHAR* in_dsql_msg_buf = NULL); static void destroy(thread_db* tdbb, dsql_req* request, bool drop); @@ -616,6 +620,7 @@ public: Firebird::Array req_msg_buffers; Firebird::string req_cursor_name; // Cursor name, if any DsqlCursor* req_cursor; // Open cursor, if any + DsqlBatch* req_batch; // Active batch, if any Firebird::GenericMap > req_user_descs; // SQLDA data type Firebird::AutoPtr req_fetch_baseline; // State of request performance counters when we reported it last time diff --git a/src/dsql/gen.cpp b/src/dsql/gen.cpp index 8847a9286c..3233e5f1c0 100644 --- a/src/dsql/gen.cpp +++ b/src/dsql/gen.cpp @@ -292,7 +292,7 @@ void GEN_request(DsqlCompilerScratch* scratch, DmlNode* node) else { GEN_port(scratch, message); - scratch->appendUChar(blr_receive); + scratch->appendUChar(blr_receive_batch); scratch->appendUChar(message->msg_number); } message = statement->getReceiveMsg(); diff --git a/src/include/firebird/FirebirdInterface.idl b/src/include/firebird/FirebirdInterface.idl index 85eff6dd16..511e5fbf8d 100644 --- a/src/include/firebird/FirebirdInterface.idl +++ b/src/include/firebird/FirebirdInterface.idl @@ -365,6 +365,10 @@ interface MessageMetadata : ReferenceCounted MetadataBuilder getBuilder(Status status); uint getMessageLength(Status status); + +version: // 3.0 => 4.0 + uint getAlignment(Status status); + uint getAlignedLength(Status status); } interface MetadataBuilder : ReferenceCounted @@ -447,8 +451,66 @@ version: // 3.0 => 4.0 // Statement execution timeout, milliseconds uint getTimeout(Status status); void setTimeout(Status status, uint timeOut); + + // Batch API + Batch createBatch(Status status, MessageMetadata inMetadata, uint parLength, const uchar* par); + + /* + Pipe createPipe(Status status, Transaction transaction, MessageMetadata inMetadata, + void* inBuffer, MessageMetadata outMetadata, uint parLength, const uchar* par); + */ } +interface Batch : ReferenceCounted +{ + const uchar VERSION1 = 1; // Tag for parameters block + + const uchar TAG_MULTIERROR = 1; // Can have >1 buffers with errors + const uchar TAG_RECORD_COUNTS = 2; // Per-record modified records accountung + const uchar TAG_BUFFER_BYTES_SIZE = 3; // Maximum possible buffer size + const uchar TAG_BLOB_POLICY = 4; // What policy is used to store blobs + const uchar TAG_DETAILED_ERRORS = 5; // How many vectors with detailed error info are stored + + const uchar BLOB_NONE = 0; // Blobs can't be used + const uchar BLOB_ID_ENGINE = 1; // Blobs are added one by one, IDs are generated by firebird + const uchar BLOB_ID_USER = 2; // Blobs are added one by one, IDs are generated by user + const uchar BLOB_STREAM = 3; // Blobs are added in a stream, IDs are generated by user + + const uint BLOB_SEGHDR_ALIGN = 2; // Alignment of segment header in the stream + + void add(Status status, uint count, const void* inBuffer); + void addBlob(Status status, uint length, const void* inBuffer, ISC_QUAD* blobId, uint parLength, const uchar* par); + void appendBlobData(Status status, uint length, const void* inBuffer); + void addBlobStream(Status status, uint length, const void* inBuffer); + void registerBlob(Status status, const ISC_QUAD* existingBlob, ISC_QUAD* blobId); + BatchCompletionState execute(Status status, Transaction transaction); + void cancel(Status status); + uint getBlobAlignment(Status status); + MessageMetadata getMetadata(Status status); + void setDefaultBpb(Status status, uint parLength, const uchar* par); +} + +interface BatchCompletionState : Disposable +{ + const int EXECUTE_FAILED = -1; // Error happened when processing record + const int SUCCESS_NO_INFO = -2; // Record update info was not collected + const uint NO_MORE_ERRORS = 0xFFFFFFFF; // Special value returned by findError() + + uint getSize(Status status); + int getState(Status status, uint pos); + uint findError(Status status, uint pos); + void getStatus(Status status, Status to, uint pos); +} + +/* +interface Pipe : ReferenceCounted +{ + uint add(Status status, uint count, void* inBuffer); + uint fetch(Status status, uint count, void* outBuffer); + void close(Status status); +} +*/ + interface Request : ReferenceCounted { void receive(Status status, int level, uint msgType, @@ -533,6 +595,15 @@ version: // 3.0 => 4.0 // Statement execution timeout, milliseconds uint getStatementTimeout(Status status); void setStatementTimeout(Status status, uint timeOut); + + // Batch API + Batch createBatch(Status status, Transaction transaction, uint stmtLength, const string sqlStmt, + uint dialect, MessageMetadata inMetadata, uint parLength, const uchar* par); + /* + Pipe createPipe(Status status, uint stmtLength, const string sqlStmt, uint dialect, + Transaction transaction, MessageMetadata inMetadata, void* inBuffer, + MessageMetadata outMetadata, uint parLength, const uchar* par); + */ } interface Service : ReferenceCounted @@ -1004,6 +1075,8 @@ interface XpbBuilder : Disposable const uint SPB_ATTACH = 2; const uint SPB_START = 3; const uint TPB = 4; + const uint BATCH = 5; + const uint BPB = 6; // removing data void clear(Status status); diff --git a/src/include/firebird/IdlFbInterfaces.h b/src/include/firebird/IdlFbInterfaces.h index a7f5ffec25..85b1d2d22e 100644 --- a/src/include/firebird/IdlFbInterfaces.h +++ b/src/include/firebird/IdlFbInterfaces.h @@ -49,6 +49,8 @@ namespace Firebird class IMetadataBuilder; class IResultSet; class IStatement; + class IBatch; + class IBatchCompletionState; class IRequest; class IEvents; class IEventBlock; @@ -1186,6 +1188,8 @@ namespace Firebird unsigned (CLOOP_CARG *getNullOffset)(IMessageMetadata* self, IStatus* status, unsigned index) throw(); IMetadataBuilder* (CLOOP_CARG *getBuilder)(IMessageMetadata* self, IStatus* status) throw(); unsigned (CLOOP_CARG *getMessageLength)(IMessageMetadata* self, IStatus* status) throw(); + unsigned (CLOOP_CARG *getAlignment)(IMessageMetadata* self, IStatus* status) throw(); + unsigned (CLOOP_CARG *getAlignedLength)(IMessageMetadata* self, IStatus* status) throw(); }; protected: @@ -1199,7 +1203,7 @@ namespace Firebird } public: - static const unsigned VERSION = 3; + static const unsigned VERSION = 4; template unsigned getCount(StatusType* status) { @@ -1320,6 +1324,34 @@ namespace Firebird StatusType::checkException(status); return ret; } + + template unsigned getAlignment(StatusType* status) + { + if (cloopVTable->version < 4) + { + StatusType::setVersionError(status, "IMessageMetadata", cloopVTable->version, 4); + StatusType::checkException(status); + return 0; + } + StatusType::clearException(status); + unsigned ret = static_cast(this->cloopVTable)->getAlignment(this, status); + StatusType::checkException(status); + return ret; + } + + template unsigned getAlignedLength(StatusType* status) + { + if (cloopVTable->version < 4) + { + StatusType::setVersionError(status, "IMessageMetadata", cloopVTable->version, 4); + StatusType::checkException(status); + return 0; + } + StatusType::clearException(status); + unsigned ret = static_cast(this->cloopVTable)->getAlignedLength(this, status); + StatusType::checkException(status); + return ret; + } }; class IMetadataBuilder : public IReferenceCounted @@ -1561,6 +1593,7 @@ namespace Firebird unsigned (CLOOP_CARG *getFlags)(IStatement* self, IStatus* status) throw(); unsigned (CLOOP_CARG *getTimeout)(IStatement* self, IStatus* status) throw(); void (CLOOP_CARG *setTimeout)(IStatement* self, IStatus* status, unsigned timeOut) throw(); + IBatch* (CLOOP_CARG *createBatch)(IStatement* self, IStatus* status, IMessageMetadata* inMetadata, unsigned parLength, const unsigned char* par) throw(); }; protected: @@ -1701,6 +1734,196 @@ namespace Firebird static_cast(this->cloopVTable)->setTimeout(this, status, timeOut); StatusType::checkException(status); } + + template IBatch* createBatch(StatusType* status, IMessageMetadata* inMetadata, unsigned parLength, const unsigned char* par) + { + if (cloopVTable->version < 4) + { + StatusType::setVersionError(status, "IStatement", cloopVTable->version, 4); + StatusType::checkException(status); + return 0; + } + StatusType::clearException(status); + IBatch* ret = static_cast(this->cloopVTable)->createBatch(this, status, inMetadata, parLength, par); + StatusType::checkException(status); + return ret; + } + }; + + class IBatch : public IReferenceCounted + { + public: + struct VTable : public IReferenceCounted::VTable + { + void (CLOOP_CARG *add)(IBatch* self, IStatus* status, unsigned count, const void* inBuffer) throw(); + void (CLOOP_CARG *addBlob)(IBatch* self, IStatus* status, unsigned length, const void* inBuffer, ISC_QUAD* blobId, unsigned parLength, const unsigned char* par) throw(); + void (CLOOP_CARG *appendBlobData)(IBatch* self, IStatus* status, unsigned length, const void* inBuffer) throw(); + void (CLOOP_CARG *addBlobStream)(IBatch* self, IStatus* status, unsigned length, const void* inBuffer) throw(); + void (CLOOP_CARG *registerBlob)(IBatch* self, IStatus* status, const ISC_QUAD* existingBlob, ISC_QUAD* blobId) throw(); + IBatchCompletionState* (CLOOP_CARG *execute)(IBatch* self, IStatus* status, ITransaction* transaction) throw(); + void (CLOOP_CARG *cancel)(IBatch* self, IStatus* status) throw(); + unsigned (CLOOP_CARG *getBlobAlignment)(IBatch* self, IStatus* status) throw(); + IMessageMetadata* (CLOOP_CARG *getMetadata)(IBatch* self, IStatus* status) throw(); + void (CLOOP_CARG *setDefaultBpb)(IBatch* self, IStatus* status, unsigned parLength, const unsigned char* par) throw(); + }; + + protected: + IBatch(DoNotInherit) + : IReferenceCounted(DoNotInherit()) + { + } + + ~IBatch() + { + } + + public: + static const unsigned VERSION = 3; + + static const unsigned char VERSION1 = 1; + static const unsigned char TAG_MULTIERROR = 1; + static const unsigned char TAG_RECORD_COUNTS = 2; + static const unsigned char TAG_BUFFER_BYTES_SIZE = 3; + static const unsigned char TAG_BLOB_POLICY = 4; + static const unsigned char TAG_DETAILED_ERRORS = 5; + static const unsigned char BLOB_NONE = 0; + static const unsigned char BLOB_ID_ENGINE = 1; + static const unsigned char BLOB_ID_USER = 2; + static const unsigned char BLOB_STREAM = 3; + static const unsigned BLOB_SEGHDR_ALIGN = 2; + + template void add(StatusType* status, unsigned count, const void* inBuffer) + { + StatusType::clearException(status); + static_cast(this->cloopVTable)->add(this, status, count, inBuffer); + StatusType::checkException(status); + } + + template void addBlob(StatusType* status, unsigned length, const void* inBuffer, ISC_QUAD* blobId, unsigned parLength, const unsigned char* par) + { + StatusType::clearException(status); + static_cast(this->cloopVTable)->addBlob(this, status, length, inBuffer, blobId, parLength, par); + StatusType::checkException(status); + } + + template void appendBlobData(StatusType* status, unsigned length, const void* inBuffer) + { + StatusType::clearException(status); + static_cast(this->cloopVTable)->appendBlobData(this, status, length, inBuffer); + StatusType::checkException(status); + } + + template void addBlobStream(StatusType* status, unsigned length, const void* inBuffer) + { + StatusType::clearException(status); + static_cast(this->cloopVTable)->addBlobStream(this, status, length, inBuffer); + StatusType::checkException(status); + } + + template void registerBlob(StatusType* status, const ISC_QUAD* existingBlob, ISC_QUAD* blobId) + { + StatusType::clearException(status); + static_cast(this->cloopVTable)->registerBlob(this, status, existingBlob, blobId); + StatusType::checkException(status); + } + + template IBatchCompletionState* execute(StatusType* status, ITransaction* transaction) + { + StatusType::clearException(status); + IBatchCompletionState* ret = static_cast(this->cloopVTable)->execute(this, status, transaction); + StatusType::checkException(status); + return ret; + } + + template void cancel(StatusType* status) + { + StatusType::clearException(status); + static_cast(this->cloopVTable)->cancel(this, status); + StatusType::checkException(status); + } + + template unsigned getBlobAlignment(StatusType* status) + { + StatusType::clearException(status); + unsigned ret = static_cast(this->cloopVTable)->getBlobAlignment(this, status); + StatusType::checkException(status); + return ret; + } + + template IMessageMetadata* getMetadata(StatusType* status) + { + StatusType::clearException(status); + IMessageMetadata* ret = static_cast(this->cloopVTable)->getMetadata(this, status); + StatusType::checkException(status); + return ret; + } + + template void setDefaultBpb(StatusType* status, unsigned parLength, const unsigned char* par) + { + StatusType::clearException(status); + static_cast(this->cloopVTable)->setDefaultBpb(this, status, parLength, par); + StatusType::checkException(status); + } + }; + + class IBatchCompletionState : public IDisposable + { + public: + struct VTable : public IDisposable::VTable + { + unsigned (CLOOP_CARG *getSize)(IBatchCompletionState* self, IStatus* status) throw(); + int (CLOOP_CARG *getState)(IBatchCompletionState* self, IStatus* status, unsigned pos) throw(); + unsigned (CLOOP_CARG *findError)(IBatchCompletionState* self, IStatus* status, unsigned pos) throw(); + void (CLOOP_CARG *getStatus)(IBatchCompletionState* self, IStatus* status, IStatus* to, unsigned pos) throw(); + }; + + protected: + IBatchCompletionState(DoNotInherit) + : IDisposable(DoNotInherit()) + { + } + + ~IBatchCompletionState() + { + } + + public: + static const unsigned VERSION = 3; + + static const int EXECUTE_FAILED = -1; + static const int SUCCESS_NO_INFO = -2; + static const unsigned NO_MORE_ERRORS = -1; + + template unsigned getSize(StatusType* status) + { + StatusType::clearException(status); + unsigned ret = static_cast(this->cloopVTable)->getSize(this, status); + StatusType::checkException(status); + return ret; + } + + template int getState(StatusType* status, unsigned pos) + { + StatusType::clearException(status); + int ret = static_cast(this->cloopVTable)->getState(this, status, pos); + StatusType::checkException(status); + return ret; + } + + template unsigned findError(StatusType* status, unsigned pos) + { + StatusType::clearException(status); + unsigned ret = static_cast(this->cloopVTable)->findError(this, status, pos); + StatusType::checkException(status); + return ret; + } + + template void getStatus(StatusType* status, IStatus* to, unsigned pos) + { + StatusType::clearException(status); + static_cast(this->cloopVTable)->getStatus(this, status, to, pos); + StatusType::checkException(status); + } }; class IRequest : public IReferenceCounted @@ -1898,6 +2121,7 @@ namespace Firebird void (CLOOP_CARG *setIdleTimeout)(IAttachment* self, IStatus* status, unsigned timeOut) throw(); unsigned (CLOOP_CARG *getStatementTimeout)(IAttachment* self, IStatus* status) throw(); void (CLOOP_CARG *setStatementTimeout)(IAttachment* self, IStatus* status, unsigned timeOut) throw(); + IBatch* (CLOOP_CARG *createBatch)(IAttachment* self, IStatus* status, ITransaction* transaction, unsigned stmtLength, const char* sqlStmt, unsigned dialect, IMessageMetadata* inMetadata, unsigned parLength, const unsigned char* par) throw(); }; protected: @@ -2102,6 +2326,20 @@ namespace Firebird static_cast(this->cloopVTable)->setStatementTimeout(this, status, timeOut); StatusType::checkException(status); } + + template IBatch* createBatch(StatusType* status, ITransaction* transaction, unsigned stmtLength, const char* sqlStmt, unsigned dialect, IMessageMetadata* inMetadata, unsigned parLength, const unsigned char* par) + { + if (cloopVTable->version < 4) + { + StatusType::setVersionError(status, "IAttachment", cloopVTable->version, 4); + StatusType::checkException(status); + return 0; + } + StatusType::clearException(status); + IBatch* ret = static_cast(this->cloopVTable)->createBatch(this, status, transaction, stmtLength, sqlStmt, dialect, inMetadata, parLength, par); + StatusType::checkException(status); + return ret; + } }; class IService : public IReferenceCounted @@ -3932,6 +4170,8 @@ namespace Firebird static const unsigned SPB_ATTACH = 2; static const unsigned SPB_START = 3; static const unsigned TPB = 4; + static const unsigned BATCH = 5; + static const unsigned BPB = 6; template void clear(StatusType* status) { @@ -7705,6 +7945,8 @@ namespace Firebird this->getNullOffset = &Name::cloopgetNullOffsetDispatcher; this->getBuilder = &Name::cloopgetBuilderDispatcher; this->getMessageLength = &Name::cloopgetMessageLengthDispatcher; + this->getAlignment = &Name::cloopgetAlignmentDispatcher; + this->getAlignedLength = &Name::cloopgetAlignedLengthDispatcher; } } vTable; @@ -7936,6 +8178,36 @@ namespace Firebird } } + static unsigned CLOOP_CARG cloopgetAlignmentDispatcher(IMessageMetadata* self, IStatus* status) throw() + { + StatusType status2(status); + + try + { + return static_cast(self)->Name::getAlignment(&status2); + } + catch (...) + { + StatusType::catchException(&status2); + return static_cast(0); + } + } + + static unsigned CLOOP_CARG cloopgetAlignedLengthDispatcher(IMessageMetadata* self, IStatus* status) throw() + { + StatusType status2(status); + + try + { + return static_cast(self)->Name::getAlignedLength(&status2); + } + catch (...) + { + StatusType::catchException(&status2); + return static_cast(0); + } + } + static void CLOOP_CARG cloopaddRefDispatcher(IReferenceCounted* self) throw() { try @@ -7990,6 +8262,8 @@ namespace Firebird virtual unsigned getNullOffset(StatusType* status, unsigned index) = 0; virtual IMetadataBuilder* getBuilder(StatusType* status) = 0; virtual unsigned getMessageLength(StatusType* status) = 0; + virtual unsigned getAlignment(StatusType* status) = 0; + virtual unsigned getAlignedLength(StatusType* status) = 0; }; template @@ -8491,6 +8765,7 @@ namespace Firebird this->getFlags = &Name::cloopgetFlagsDispatcher; this->getTimeout = &Name::cloopgetTimeoutDispatcher; this->setTimeout = &Name::cloopsetTimeoutDispatcher; + this->createBatch = &Name::cloopcreateBatchDispatcher; } } vTable; @@ -8688,6 +8963,21 @@ namespace Firebird } } + static IBatch* CLOOP_CARG cloopcreateBatchDispatcher(IStatement* self, IStatus* status, IMessageMetadata* inMetadata, unsigned parLength, const unsigned char* par) throw() + { + StatusType status2(status); + + try + { + return static_cast(self)->Name::createBatch(&status2, inMetadata, parLength, par); + } + catch (...) + { + StatusType::catchException(&status2); + return static_cast(0); + } + } + static void CLOOP_CARG cloopaddRefDispatcher(IReferenceCounted* self) throw() { try @@ -8740,6 +9030,347 @@ namespace Firebird virtual unsigned getFlags(StatusType* status) = 0; virtual unsigned getTimeout(StatusType* status) = 0; virtual void setTimeout(StatusType* status, unsigned timeOut) = 0; + virtual IBatch* createBatch(StatusType* status, IMessageMetadata* inMetadata, unsigned parLength, const unsigned char* par) = 0; + }; + + template + class IBatchBaseImpl : public Base + { + public: + typedef IBatch Declaration; + + IBatchBaseImpl(DoNotInherit = DoNotInherit()) + { + static struct VTableImpl : Base::VTable + { + VTableImpl() + { + this->version = Base::VERSION; + this->addRef = &Name::cloopaddRefDispatcher; + this->release = &Name::cloopreleaseDispatcher; + this->add = &Name::cloopaddDispatcher; + this->addBlob = &Name::cloopaddBlobDispatcher; + this->appendBlobData = &Name::cloopappendBlobDataDispatcher; + this->addBlobStream = &Name::cloopaddBlobStreamDispatcher; + this->registerBlob = &Name::cloopregisterBlobDispatcher; + this->execute = &Name::cloopexecuteDispatcher; + this->cancel = &Name::cloopcancelDispatcher; + this->getBlobAlignment = &Name::cloopgetBlobAlignmentDispatcher; + this->getMetadata = &Name::cloopgetMetadataDispatcher; + this->setDefaultBpb = &Name::cloopsetDefaultBpbDispatcher; + } + } vTable; + + this->cloopVTable = &vTable; + } + + static void CLOOP_CARG cloopaddDispatcher(IBatch* self, IStatus* status, unsigned count, const void* inBuffer) throw() + { + StatusType status2(status); + + try + { + static_cast(self)->Name::add(&status2, count, inBuffer); + } + catch (...) + { + StatusType::catchException(&status2); + } + } + + static void CLOOP_CARG cloopaddBlobDispatcher(IBatch* self, IStatus* status, unsigned length, const void* inBuffer, ISC_QUAD* blobId, unsigned parLength, const unsigned char* par) throw() + { + StatusType status2(status); + + try + { + static_cast(self)->Name::addBlob(&status2, length, inBuffer, blobId, parLength, par); + } + catch (...) + { + StatusType::catchException(&status2); + } + } + + static void CLOOP_CARG cloopappendBlobDataDispatcher(IBatch* self, IStatus* status, unsigned length, const void* inBuffer) throw() + { + StatusType status2(status); + + try + { + static_cast(self)->Name::appendBlobData(&status2, length, inBuffer); + } + catch (...) + { + StatusType::catchException(&status2); + } + } + + static void CLOOP_CARG cloopaddBlobStreamDispatcher(IBatch* self, IStatus* status, unsigned length, const void* inBuffer) throw() + { + StatusType status2(status); + + try + { + static_cast(self)->Name::addBlobStream(&status2, length, inBuffer); + } + catch (...) + { + StatusType::catchException(&status2); + } + } + + static void CLOOP_CARG cloopregisterBlobDispatcher(IBatch* self, IStatus* status, const ISC_QUAD* existingBlob, ISC_QUAD* blobId) throw() + { + StatusType status2(status); + + try + { + static_cast(self)->Name::registerBlob(&status2, existingBlob, blobId); + } + catch (...) + { + StatusType::catchException(&status2); + } + } + + static IBatchCompletionState* CLOOP_CARG cloopexecuteDispatcher(IBatch* self, IStatus* status, ITransaction* transaction) throw() + { + StatusType status2(status); + + try + { + return static_cast(self)->Name::execute(&status2, transaction); + } + catch (...) + { + StatusType::catchException(&status2); + return static_cast(0); + } + } + + static void CLOOP_CARG cloopcancelDispatcher(IBatch* self, IStatus* status) throw() + { + StatusType status2(status); + + try + { + static_cast(self)->Name::cancel(&status2); + } + catch (...) + { + StatusType::catchException(&status2); + } + } + + static unsigned CLOOP_CARG cloopgetBlobAlignmentDispatcher(IBatch* self, IStatus* status) throw() + { + StatusType status2(status); + + try + { + return static_cast(self)->Name::getBlobAlignment(&status2); + } + catch (...) + { + StatusType::catchException(&status2); + return static_cast(0); + } + } + + static IMessageMetadata* CLOOP_CARG cloopgetMetadataDispatcher(IBatch* self, IStatus* status) throw() + { + StatusType status2(status); + + try + { + return static_cast(self)->Name::getMetadata(&status2); + } + catch (...) + { + StatusType::catchException(&status2); + return static_cast(0); + } + } + + static void CLOOP_CARG cloopsetDefaultBpbDispatcher(IBatch* self, IStatus* status, unsigned parLength, const unsigned char* par) throw() + { + StatusType status2(status); + + try + { + static_cast(self)->Name::setDefaultBpb(&status2, parLength, par); + } + catch (...) + { + StatusType::catchException(&status2); + } + } + + static void CLOOP_CARG cloopaddRefDispatcher(IReferenceCounted* self) throw() + { + try + { + static_cast(self)->Name::addRef(); + } + catch (...) + { + StatusType::catchException(0); + } + } + + static int CLOOP_CARG cloopreleaseDispatcher(IReferenceCounted* self) throw() + { + try + { + return static_cast(self)->Name::release(); + } + catch (...) + { + StatusType::catchException(0); + return static_cast(0); + } + } + }; + + template > > > > + class IBatchImpl : public IBatchBaseImpl + { + protected: + IBatchImpl(DoNotInherit = DoNotInherit()) + { + } + + public: + virtual ~IBatchImpl() + { + } + + virtual void add(StatusType* status, unsigned count, const void* inBuffer) = 0; + virtual void addBlob(StatusType* status, unsigned length, const void* inBuffer, ISC_QUAD* blobId, unsigned parLength, const unsigned char* par) = 0; + virtual void appendBlobData(StatusType* status, unsigned length, const void* inBuffer) = 0; + virtual void addBlobStream(StatusType* status, unsigned length, const void* inBuffer) = 0; + virtual void registerBlob(StatusType* status, const ISC_QUAD* existingBlob, ISC_QUAD* blobId) = 0; + virtual IBatchCompletionState* execute(StatusType* status, ITransaction* transaction) = 0; + virtual void cancel(StatusType* status) = 0; + virtual unsigned getBlobAlignment(StatusType* status) = 0; + virtual IMessageMetadata* getMetadata(StatusType* status) = 0; + virtual void setDefaultBpb(StatusType* status, unsigned parLength, const unsigned char* par) = 0; + }; + + template + class IBatchCompletionStateBaseImpl : public Base + { + public: + typedef IBatchCompletionState Declaration; + + IBatchCompletionStateBaseImpl(DoNotInherit = DoNotInherit()) + { + static struct VTableImpl : Base::VTable + { + VTableImpl() + { + this->version = Base::VERSION; + this->dispose = &Name::cloopdisposeDispatcher; + this->getSize = &Name::cloopgetSizeDispatcher; + this->getState = &Name::cloopgetStateDispatcher; + this->findError = &Name::cloopfindErrorDispatcher; + this->getStatus = &Name::cloopgetStatusDispatcher; + } + } vTable; + + this->cloopVTable = &vTable; + } + + static unsigned CLOOP_CARG cloopgetSizeDispatcher(IBatchCompletionState* self, IStatus* status) throw() + { + StatusType status2(status); + + try + { + return static_cast(self)->Name::getSize(&status2); + } + catch (...) + { + StatusType::catchException(&status2); + return static_cast(0); + } + } + + static int CLOOP_CARG cloopgetStateDispatcher(IBatchCompletionState* self, IStatus* status, unsigned pos) throw() + { + StatusType status2(status); + + try + { + return static_cast(self)->Name::getState(&status2, pos); + } + catch (...) + { + StatusType::catchException(&status2); + return static_cast(0); + } + } + + static unsigned CLOOP_CARG cloopfindErrorDispatcher(IBatchCompletionState* self, IStatus* status, unsigned pos) throw() + { + StatusType status2(status); + + try + { + return static_cast(self)->Name::findError(&status2, pos); + } + catch (...) + { + StatusType::catchException(&status2); + return static_cast(0); + } + } + + static void CLOOP_CARG cloopgetStatusDispatcher(IBatchCompletionState* self, IStatus* status, IStatus* to, unsigned pos) throw() + { + StatusType status2(status); + + try + { + static_cast(self)->Name::getStatus(&status2, to, pos); + } + catch (...) + { + StatusType::catchException(&status2); + } + } + + static void CLOOP_CARG cloopdisposeDispatcher(IDisposable* self) throw() + { + try + { + static_cast(self)->Name::dispose(); + } + catch (...) + { + StatusType::catchException(0); + } + } + }; + + template > > > > + class IBatchCompletionStateImpl : public IBatchCompletionStateBaseImpl + { + protected: + IBatchCompletionStateImpl(DoNotInherit = DoNotInherit()) + { + } + + public: + virtual ~IBatchCompletionStateImpl() + { + } + + virtual unsigned getSize(StatusType* status) = 0; + virtual int getState(StatusType* status, unsigned pos) = 0; + virtual unsigned findError(StatusType* status, unsigned pos) = 0; + virtual void getStatus(StatusType* status, IStatus* to, unsigned pos) = 0; }; template @@ -9168,6 +9799,7 @@ namespace Firebird this->setIdleTimeout = &Name::cloopsetIdleTimeoutDispatcher; this->getStatementTimeout = &Name::cloopgetStatementTimeoutDispatcher; this->setStatementTimeout = &Name::cloopsetStatementTimeoutDispatcher; + this->createBatch = &Name::cloopcreateBatchDispatcher; } } vTable; @@ -9494,6 +10126,21 @@ namespace Firebird } } + static IBatch* CLOOP_CARG cloopcreateBatchDispatcher(IAttachment* self, IStatus* status, ITransaction* transaction, unsigned stmtLength, const char* sqlStmt, unsigned dialect, IMessageMetadata* inMetadata, unsigned parLength, const unsigned char* par) throw() + { + StatusType status2(status); + + try + { + return static_cast(self)->Name::createBatch(&status2, transaction, stmtLength, sqlStmt, dialect, inMetadata, parLength, par); + } + catch (...) + { + StatusType::catchException(&status2); + return static_cast(0); + } + } + static void CLOOP_CARG cloopaddRefDispatcher(IReferenceCounted* self) throw() { try @@ -9555,6 +10202,7 @@ namespace Firebird virtual void setIdleTimeout(StatusType* status, unsigned timeOut) = 0; virtual unsigned getStatementTimeout(StatusType* status) = 0; virtual void setStatementTimeout(StatusType* status, unsigned timeOut) = 0; + virtual IBatch* createBatch(StatusType* status, ITransaction* transaction, unsigned stmtLength, const char* sqlStmt, unsigned dialect, IMessageMetadata* inMetadata, unsigned parLength, const unsigned char* par) = 0; }; template diff --git a/src/jrd/Attachment.h b/src/jrd/Attachment.h index 876ce45ce0..95d04ac64e 100644 --- a/src/jrd/Attachment.h +++ b/src/jrd/Attachment.h @@ -312,7 +312,7 @@ public: Firebird::PathName att_filename; // alias used to attach the database const Firebird::TimeStamp att_timestamp; // Connection date and time Firebird::StringMap att_context_vars; // Context variables for the connection - Firebird::Stack ddlTriggersContext; // Context variables for DDL trigger event + Firebird::Stack ddlTriggersContext; // Context variables for DDL trigger event Firebird::string att_network_protocol; // Network protocol used by client for connection Firebird::string att_remote_address; // Protocol-specific address of remote client SLONG att_remote_pid; // Process id of remote client diff --git a/src/jrd/EngineInterface.h b/src/jrd/EngineInterface.h index 7465a171b2..3576bae22d 100644 --- a/src/jrd/EngineInterface.h +++ b/src/jrd/EngineInterface.h @@ -34,6 +34,7 @@ namespace Jrd { class blb; class jrd_tra; class DsqlCursor; +class DsqlBatch; class dsql_req; class JrdStatement; class StableAttachmentPart; @@ -181,6 +182,44 @@ private: void freeEngineData(Firebird::CheckStatusWrapper* status); }; +class JBatch FB_FINAL : + public Firebird::RefCntIface > +{ +public: + // IBatch implementation + int release(); + void add(Firebird::CheckStatusWrapper* status, unsigned count, const void* inBuffer); + void addBlob(Firebird::CheckStatusWrapper* status, unsigned length, const void* inBuffer, ISC_QUAD* blobId, + unsigned parLength, const unsigned char* par); + void appendBlobData(Firebird::CheckStatusWrapper* status, unsigned length, const void* inBuffer); + void addBlobStream(Firebird::CheckStatusWrapper* status, unsigned length, const void* inBuffer); + void registerBlob(Firebird::CheckStatusWrapper* status, const ISC_QUAD* existingBlob, ISC_QUAD* blobId); + Firebird::IBatchCompletionState* execute(Firebird::CheckStatusWrapper* status, Firebird::ITransaction* transaction); + void cancel(Firebird::CheckStatusWrapper* status); + unsigned getBlobAlignment(Firebird::CheckStatusWrapper* status); + Firebird::IMessageMetadata* getMetadata(Firebird::CheckStatusWrapper* status); + void setDefaultBpb(Firebird::CheckStatusWrapper* status, unsigned parLength, const unsigned char* par); + +public: + JBatch(DsqlBatch* handle, JStatement* aStatement); + + StableAttachmentPart* getAttachment(); + + DsqlBatch* getHandle() throw() + { + return batch; + } + + void resetHandle() + { + batch = NULL; + } + +private: + DsqlBatch* batch; + Firebird::RefPtr statement; +}; + class JStatement FB_FINAL : public Firebird::RefCntIface > { @@ -207,6 +246,8 @@ public: unsigned int getTimeout(Firebird::CheckStatusWrapper* status); void setTimeout(Firebird::CheckStatusWrapper* status, unsigned int timeOut); + JBatch* createBatch(Firebird::CheckStatusWrapper* status, Firebird::IMessageMetadata* inMetadata, + unsigned parLength, const unsigned char* par); public: JStatement(dsql_req* handle, StableAttachmentPart* sa, Firebird::Array& meta); @@ -352,6 +393,9 @@ public: void setIdleTimeout(Firebird::CheckStatusWrapper* status, unsigned int timeOut); unsigned int getStatementTimeout(Firebird::CheckStatusWrapper* status); void setStatementTimeout(Firebird::CheckStatusWrapper* status, unsigned int timeOut); + Firebird::IBatch* createBatch(Firebird::CheckStatusWrapper* status, Firebird::ITransaction* transaction, + unsigned stmtLength, const char* sqlStmt, unsigned dialect, + Firebird::IMessageMetadata* inMetadata, unsigned parLength, const unsigned char* par); public: explicit JAttachment(StableAttachmentPart* js); diff --git a/src/jrd/SysFunction.cpp b/src/jrd/SysFunction.cpp index e272e4cd29..4eb18febe6 100644 --- a/src/jrd/SysFunction.cpp +++ b/src/jrd/SysFunction.cpp @@ -2529,44 +2529,44 @@ dsc* evlGetContext(thread_db* tdbb, const SysFunction*, const NestValueArray& ar if (!attachment->ddlTriggersContext.hasData()) status_exception::raise(Arg::Gds(isc_sysf_invalid_trig_namespace)); - const DdlTriggerContext& context = Stack::const_iterator( + const DdlTriggerContext* context = Stack::const_iterator( attachment->ddlTriggersContext).object(); if (nameStr == EVENT_TYPE_NAME) - resultStr = context.eventType; + resultStr = context->eventType; else if (nameStr == OBJECT_TYPE_NAME) - resultStr = context.objectType; + resultStr = context->objectType; else if (nameStr == DDL_EVENT_NAME) - resultStr = context.eventType + " " + context.objectType; + resultStr = context->eventType + " " + context->objectType; else if (nameStr == OBJECT_NAME) { - resultStr = context.objectName.c_str(); + resultStr = context->objectName.c_str(); resultType = ttype_metadata; } else if (nameStr == OLD_OBJECT_NAME) { - if (context.oldObjectName.isEmpty()) + if (context->oldObjectName.isEmpty()) return NULL; - resultStr = context.oldObjectName.c_str(); + resultStr = context->oldObjectName.c_str(); resultType = ttype_metadata; } else if (nameStr == NEW_OBJECT_NAME) { - if (context.newObjectName.isEmpty()) + if (context->newObjectName.isEmpty()) return NULL; - resultStr = context.newObjectName.c_str(); + resultStr = context->newObjectName.c_str(); resultType = ttype_metadata; } else if (nameStr == SQL_TEXT_NAME) { - if (context.sqlText.isEmpty()) + if (context->sqlText.isEmpty()) return NULL; blb* blob = blb::create(tdbb, transaction, &impure->vlu_misc.vlu_bid); - blob->BLB_put_data(tdbb, reinterpret_cast(context.sqlText.c_str()), - context.sqlText.length()); + blob->BLB_put_data(tdbb, reinterpret_cast(context->sqlText.c_str()), + context->sqlText.length()); blob->BLB_close(tdbb); dsc result; diff --git a/src/jrd/blr.h b/src/jrd/blr.h index c87854314f..6e9a3f744f 100644 --- a/src/jrd/blr.h +++ b/src/jrd/blr.h @@ -129,8 +129,9 @@ #define blr_maximum (unsigned char)29 #define blr_minimum (unsigned char)30 #define blr_total (unsigned char)31 +#define blr_receive_batch (unsigned char)32 -// unused codes: 32..33 +// unused code: 33 #define blr_add (unsigned char)34 #define blr_subtract (unsigned char)35 diff --git a/src/jrd/ibase.h b/src/jrd/ibase.h index 76bb1cc842..4be20b3b52 100644 --- a/src/jrd/ibase.h +++ b/src/jrd/ibase.h @@ -479,6 +479,10 @@ ISC_STATUS ISC_EXPORT fb_dsql_set_timeout(ISC_STATUS*, isc_stmt_handle*, ISC_ULONG); +/*ISC_STATUS ISC_EXPORT fb_get_statement_interface(ISC_STATUS*, + FB_API_HANDLE*, + void**); + */ void ISC_EXPORT isc_encode_date(const void*, ISC_QUAD*); diff --git a/src/jrd/inf_pub.h b/src/jrd/inf_pub.h index 2c49145656..bf56df6269 100644 --- a/src/jrd/inf_pub.h +++ b/src/jrd/inf_pub.h @@ -430,6 +430,7 @@ enum info_db_provider #define isc_info_sql_stmt_flags 27 #define isc_info_sql_stmt_timeout_user 28 #define isc_info_sql_stmt_timeout_run 29 +#define isc_info_sql_stmt_blob_align 30 /*********************************/ /* SQL information return values */ diff --git a/src/jrd/jrd.cpp b/src/jrd/jrd.cpp index 4cee5e131f..12a744f0a0 100644 --- a/src/jrd/jrd.cpp +++ b/src/jrd/jrd.cpp @@ -132,6 +132,7 @@ #include "../dsql/dsql.h" #include "../dsql/dsql_proto.h" +#include "../dsql/DsqlBatch.h" using namespace Jrd; using namespace Firebird; @@ -650,6 +651,14 @@ namespace validateHandle(tdbb, cursor->getAttachment()); } + inline void validateHandle(thread_db* tdbb, DsqlBatch* const batch) + { + if (!batch) + status_exception::raise(Arg::Gds(isc_bad_req_handle)); // isc_bad_batch_handle !!!!!!!!!! + + validateHandle(tdbb, batch->getAttachment()); + } + class AttachmentHolder { public: @@ -1158,12 +1167,21 @@ ISC_STATUS transliterateException(thread_db* tdbb, const Exception& ex, FbStatus attachment->att_trace_manager->event_error(&conn, &traceStatus, func); } + JRD_transliterate(tdbb, vector); + return vector->getErrors()[1]; +} + + +// Transliterate status vector to the client charset. +void JRD_transliterate(thread_db* tdbb, Firebird::IStatus* vector) throw() +{ + Jrd::Attachment* attachment = tdbb->getAttachment(); USHORT charSet; if (!attachment || (charSet = attachment->att_client_charset) == CS_METADATA || charSet == CS_NONE) { - return vector->getErrors()[1]; + return; } const ISC_STATUS* const vectorStart = vector->getErrors(); @@ -1236,12 +1254,10 @@ ISC_STATUS transliterateException(thread_db* tdbb, const Exception& ex, FbStatus } catch (...) { - ex.stuffException(vector); - return vector->getErrors()[1]; + return; } vector->setErrors2(newVector.getCount() - 1, newVector.begin()); - return vector->getErrors()[1]; } @@ -4793,6 +4809,21 @@ ITransaction* JAttachment::execute(CheckStatusWrapper* user_status, ITransaction } +IBatch* JAttachment::createBatch(CheckStatusWrapper* status, ITransaction* transaction, + unsigned stmtLength, const char* sqlStmt, unsigned dialect, + IMessageMetadata* inMetadata, unsigned parLength, const unsigned char* par) +{ + RefPtr tmpStatement(REF_NO_INCR, prepare(status, transaction, stmtLength, sqlStmt, + dialect, 0)); + if (status->getState() & IStatus::STATE_ERRORS) + { + return NULL; + } + + return tmpStatement->createBatch(status, inMetadata, parLength, par); +} + + int JResultSet::fetchNext(CheckStatusWrapper* user_status, void* buffer) { try @@ -5472,6 +5503,369 @@ void JStatement::setTimeout(CheckStatusWrapper* user_status, unsigned int timeOu } +JBatch* JStatement::createBatch(Firebird::CheckStatusWrapper* status, Firebird::IMessageMetadata* inMetadata, + unsigned parLength, const unsigned char* par) +{ + JBatch* batch = NULL; + + try + { + EngineContextHolder tdbb(status, this, FB_FUNCTION); + check_database(tdbb); + + try + { + RefPtr defaultIn; + if (!inMetadata) + { + defaultIn.assignRefNoIncr(metadata.getInputMetadata()); + if (defaultIn) + { + inMetadata = defaultIn; + } + } + + DsqlBatch* const b = DsqlBatch::open(tdbb, getHandle(), inMetadata, parLength, par); + + batch = FB_NEW JBatch(b, this); + batch->addRef(); + b->setInterfacePtr(batch); + } + catch (const Exception& ex) + { + transliterateException(tdbb, ex, status, "JStatement::createBatch"); + return NULL; + } + + trace_warning(tdbb, status, "JStatement::createBatch"); + } + catch (const Exception& ex) + { + ex.stuffException(status); + return NULL; + } + + successful_completion(status); + return batch; +} + + +JBatch::JBatch(DsqlBatch* handle, JStatement* aStatement) + : batch(handle), + statement(aStatement) +{ } + + +StableAttachmentPart* JBatch::getAttachment() +{ + return statement->getAttachment(); +} + + +int JBatch::release() +{ + if (--refCounter != 0) + return 1; + + if (batch) + delete batch; + + delete this; + return 0; +} + + +void JBatch::add(CheckStatusWrapper* status, unsigned count, const void* inBuffer) +{ + try + { + EngineContextHolder tdbb(status, this, FB_FUNCTION); + check_database(tdbb); + + try + { + DsqlBatch* b = getHandle(); + b->add(tdbb, count, inBuffer); + } + catch (const Exception& ex) + { + transliterateException(tdbb, ex, status, "JBatch::add"); + return; + } + + trace_warning(tdbb, status, "JBatch::add"); + } + catch (const Exception& ex) + { + ex.stuffException(status); + return; + } + + successful_completion(status); +} + + +void JBatch::addBlob(CheckStatusWrapper* status, unsigned length, const void* inBuffer, ISC_QUAD* blobId, + unsigned parLength, const unsigned char* par) +{ + try + { + EngineContextHolder tdbb(status, this, FB_FUNCTION); + check_database(tdbb); + + try + { + DsqlBatch* b = getHandle(); + b->addBlob(tdbb, length, inBuffer, blobId, parLength, par); + } + catch (const Exception& ex) + { + transliterateException(tdbb, ex, status, "JBatch::addBlob"); + return; + } + + trace_warning(tdbb, status, "JBatch::addBlob"); + } + catch (const Exception& ex) + { + ex.stuffException(status); + return; + } + + successful_completion(status); +} + + +void JBatch::appendBlobData(CheckStatusWrapper* status, unsigned length, const void* inBuffer) +{ + try + { + EngineContextHolder tdbb(status, this, FB_FUNCTION); + check_database(tdbb); + + try + { + DsqlBatch* b = getHandle(); + b->appendBlobData(tdbb, length, inBuffer); + } + catch (const Exception& ex) + { + transliterateException(tdbb, ex, status, "JBatch::appendBlobData"); + return; + } + + trace_warning(tdbb, status, "JBatch::appendBlobData"); + } + catch (const Exception& ex) + { + ex.stuffException(status); + return; + } + + successful_completion(status); +} + + +void JBatch::addBlobStream(CheckStatusWrapper* status, unsigned length, const void* inBuffer) +{ + try + { + EngineContextHolder tdbb(status, this, FB_FUNCTION); + check_database(tdbb); + + try + { + DsqlBatch* b = getHandle(); + b->addBlobStream(tdbb, length, inBuffer); + } + catch (const Exception& ex) + { + transliterateException(tdbb, ex, status, "JBatch::addBlobStream"); + return; + } + + trace_warning(tdbb, status, "JBatch::addBlobStream"); + } + catch (const Exception& ex) + { + ex.stuffException(status); + return; + } + + successful_completion(status); +} + + +void JBatch::setDefaultBpb(CheckStatusWrapper* status, unsigned parLength, const unsigned char* par) +{ + try + { + EngineContextHolder tdbb(status, this, FB_FUNCTION); + check_database(tdbb); + + try + { + DsqlBatch* b = getHandle(); + b->setDefaultBpb(tdbb, parLength, par); + } + catch (const Exception& ex) + { + transliterateException(tdbb, ex, status, "JBatch::setDefaultBpb"); + return; + } + + trace_warning(tdbb, status, "JBatch::setDefaultBpb"); + } + catch (const Exception& ex) + { + ex.stuffException(status); + return; + } + + successful_completion(status); +} + + +unsigned JBatch::getBlobAlignment(CheckStatusWrapper*) +{ + return DsqlBatch::BLOB_STREAM_ALIGN; +} + + +IMessageMetadata* JBatch::getMetadata(CheckStatusWrapper* status) +{ + IMessageMetadata* meta; + try + { + EngineContextHolder tdbb(status, this, FB_FUNCTION); + check_database(tdbb); + + try + { + DsqlBatch* b = getHandle(); + meta = b->getMetadata(tdbb); + } + catch (const Exception& ex) + { + transliterateException(tdbb, ex, status, "JBatch::getMetadata"); + return NULL; + } + + trace_warning(tdbb, status, "JBatch::getMetadata"); + } + catch (const Exception& ex) + { + ex.stuffException(status); + return NULL; + } + + successful_completion(status); + return meta; +} + + +void JBatch::registerBlob(CheckStatusWrapper* status, const ISC_QUAD* existingBlob, ISC_QUAD* blobId) +{ + try + { + EngineContextHolder tdbb(status, this, FB_FUNCTION); + check_database(tdbb); + + try + { + DsqlBatch* b = getHandle(); + b->registerBlob(tdbb, existingBlob, blobId); + } + catch (const Exception& ex) + { + transliterateException(tdbb, ex, status, "JBatch::registerBlob"); + return; + } + + trace_warning(tdbb, status, "JBatch::registerBlob"); + } + catch (const Exception& ex) + { + ex.stuffException(status); + return; + } + + successful_completion(status); +} + + +IBatchCompletionState* JBatch::execute(CheckStatusWrapper* status, ITransaction* transaction) +{ + IBatchCompletionState* cs; + try + { + EngineContextHolder tdbb(status, this, FB_FUNCTION); + + jrd_tra* tra = nullptr; + if (transaction) + { + JTransaction* jt = getAttachment()->getTransactionInterface(status, transaction); + if (jt) + tra = jt->getHandle(); + } + + validateHandle(tdbb, tra); + check_database(tdbb); + + try + { + DsqlBatch* b = getHandle(); + cs = b->execute(tdbb); + } + catch (const Exception& ex) + { + transliterateException(tdbb, ex, status, "JBatch::execute"); + return NULL; + } + + trace_warning(tdbb, status, "JBatch::execute"); + } + catch (const Exception& ex) + { + ex.stuffException(status); + return NULL; + } + + successful_completion(status); + return cs; +} + + +void JBatch::cancel(CheckStatusWrapper* status) +{ + try + { + EngineContextHolder tdbb(status, this, FB_FUNCTION); + check_database(tdbb); + + try + { + DsqlBatch* b = getHandle(); + b->cancel(tdbb); + } + catch (const Exception& ex) + { + transliterateException(tdbb, ex, status, "JBatch::cancel"); + return; + } + + trace_warning(tdbb, status, "JBatch::cancel"); + } + catch (const Exception& ex) + { + ex.stuffException(status); + return; + } + + successful_completion(status); +} + + void JAttachment::ping(CheckStatusWrapper* user_status) { /************************************** diff --git a/src/jrd/jrd_proto.h b/src/jrd/jrd_proto.h index 39ed60d53e..7f303a2544 100644 --- a/src/jrd/jrd_proto.h +++ b/src/jrd/jrd_proto.h @@ -27,6 +27,7 @@ #include "../common/classes/fb_string.h" #include "../common/classes/objects_array.h" +#include "../jrd/status.h" namespace Jrd { class Database; @@ -80,6 +81,7 @@ void JRD_shutdown_attachment(Jrd::Attachment* attachment); void JRD_shutdown_attachments(Jrd::Database* dbb); void JRD_cancel_operation(Jrd::thread_db* tdbb, Jrd::Attachment* attachment, int option); void JRD_make_role_name(Firebird::MetaName& userIdRole, const int dialect); +void JRD_transliterate(Jrd::thread_db* tdbb, Firebird::IStatus* vector) throw(); bool JRD_shutdown_database(Jrd::Database* dbb, const unsigned flags = 0); // JRD_shutdown_database() flags diff --git a/src/jrd/req.h b/src/jrd/req.h index 47bedf1065..9678bfa55d 100644 --- a/src/jrd/req.h +++ b/src/jrd/req.h @@ -284,6 +284,7 @@ public: } req_operation; // operation for next node StatusXcp req_last_xcp; // last known exception + bool req_batch; template T* getImpure(unsigned offset) { diff --git a/src/misc/codes.epp b/src/misc/codes.epp index 1380ac6cfe..2bbbd13919 100644 --- a/src/misc/codes.epp +++ b/src/misc/codes.epp @@ -430,7 +430,7 @@ static void build_iberror_h() if (last_code + 1 != N.CODE && N.FAC_CODE == 0) { fprintf(stderr, - "Warning: missing codes between %d and %d (exclusive)\n", + "Warning: 1 missing codes between %d and %d (exclusive)\n", last_code, (int) N.CODE); } last_code = N.CODE; @@ -474,7 +474,7 @@ static void build_iberror_h() if (last_code + 1 != N.CODE && N.FAC_CODE == 0) { fprintf(stderr, - "Warning: missing codes between %d and %d (exclusive)\n", + "Warning: 2 missing codes between %d and %d (exclusive)\n", last_code, (int) N.CODE); } last_code = N.CODE; @@ -539,7 +539,7 @@ static void build_iberror_h() // if (last_code + 1 != N.CODE && N.FAC_CODE == 0) // { // fprintf(stderr, -// "Warning: missing codes between %d and %d (exclusive)\n", +// "Warning: 3 missing codes between %d and %d (exclusive)\n", // last_code, (int) N.CODE); // } // last_code = N.CODE; @@ -610,7 +610,7 @@ static void build_iberror_h() // if (last_code + 1 != N.CODE && N.FAC_CODE == 0) // { // fprintf(stderr, -// "Warning: missing codes between %d and %d (exclusive)\n", +// "Warning: 4 missing codes between %d and %d (exclusive)\n", // last_code, (int) N.CODE); // } // last_code = N.CODE; @@ -820,7 +820,7 @@ static void build_other_headers() if (last_code + 1 != N.CODE && N.FAC_CODE == 0) { fprintf(stderr, - "Warning: missing codes between %d and %d (exclusive)\n", + "Warning: 5 missing codes between %d and %d (exclusive)\n", last_code, (int) N.CODE); } last_code = N.CODE; diff --git a/src/remote/client/interface.cpp b/src/remote/client/interface.cpp index b6784f6914..3bb5ba0a94 100644 --- a/src/remote/client/interface.cpp +++ b/src/remote/client/interface.cpp @@ -55,6 +55,7 @@ #include "../yvalve/gds_proto.h" #include "../common/isc_f_proto.h" #include "../common/classes/ClumpletWriter.h" +#include "../common/classes/BatchCompletionState.h" #include "../common/config/config.h" #include "../common/utils_proto.h" #include "../common/classes/DbImplementation.h" @@ -68,9 +69,8 @@ #include "../auth/SecureRemotePassword/client/SrpClient.h" #include "../auth/trusted/AuthSspi.h" #include "../plugins/crypt/arc4/Arc4.h" - #include "BlrFromMessage.h" - +#include "../dsql/DsqlBatch.h" #ifdef HAVE_UNISTD_H #include @@ -299,6 +299,218 @@ int ResultSet::release() return 0; } +class Batch FB_FINAL : public RefCntIface > +{ +public: + Batch(Statement* s, IMessageMetadata* inFmt, unsigned parLength, const unsigned char* par); + + // IResultSet implementation + int release(); + void add(Firebird::CheckStatusWrapper* status, unsigned count, const void* inBuffer); + void addBlob(Firebird::CheckStatusWrapper* status, unsigned length, const void* inBuffer, ISC_QUAD* blobId, + unsigned parLength, const unsigned char* par); + void appendBlobData(Firebird::CheckStatusWrapper* status, unsigned length, const void* inBuffer); + void addBlobStream(Firebird::CheckStatusWrapper* status, unsigned length, const void* inBuffer); + void registerBlob(Firebird::CheckStatusWrapper* status, const ISC_QUAD* existingBlob, ISC_QUAD* blobId); + Firebird::IBatchCompletionState* execute(Firebird::CheckStatusWrapper* status, Firebird::ITransaction* transaction); + void cancel(Firebird::CheckStatusWrapper* status); + unsigned getBlobAlignment(Firebird::CheckStatusWrapper* status); + void setDefaultBpb(Firebird::CheckStatusWrapper* status, unsigned parLength, const unsigned char* par); + Firebird::IMessageMetadata* getMetadata(Firebird::CheckStatusWrapper* status); + +private: + void freeClientData(CheckStatusWrapper* status, bool force = false); + void releaseStatement(); + void setBlobAlignment(); + + void genBlobId(ISC_QUAD* blobId) + { + if (++genId.gds_quad_low == 0) + ++genId.gds_quad_high; + memcpy(blobId, &genId, sizeof(genId)); + } + + bool batchHasData() + { + return batchActive; + } + + // working with message stream buffer + void putMessageData(ULONG count, const void* p) + { + fb_assert(messageStreamBuffer); + + const UCHAR* ptr = reinterpret_cast(p); + + while(count) + { + ULONG remainSpace = messageBufferSize - messageStream; + ULONG step = MIN(count, remainSpace); + if (step == messageBufferSize) + { + // direct packet sent + sendMessagePacket(step, ptr); + } + else + { + // use buffer + memcpy(&messageStreamBuffer[messageStream * alignedSize], ptr, step * alignedSize); + messageStream += step; + if (messageStream == messageBufferSize) + { + sendMessagePacket(messageBufferSize, messageStreamBuffer); + messageStream = 0; + } + } + + count -= step; + ptr += step * alignedSize; + } + } + + // working with blob stream buffer + void newBlob() + { + alignBlobBuffer(blobAlign); + + fb_assert(blobStream - blobStreamBuffer <= blobBufferSize); + ULONG space = blobBufferSize - (blobStream - blobStreamBuffer); + if (space < Rsr::BatchStream::SIZEOF_BLOB_HEAD) + { + sendBlobPacket(blobStream - blobStreamBuffer, blobStreamBuffer); + blobStream = blobStreamBuffer; + } + } + + void alignBlobBuffer(unsigned alignment, ULONG* bs = NULL) + { + FB_UINT64 zeroFill = 0; + UCHAR* newPointer = FB_ALIGN(blobStream, alignment); + ULONG align = FB_ALIGN(blobStream, alignment) - blobStream; + putBlobData(align, &zeroFill); + if (bs) + *bs += align; + } + + void putBlobData(ULONG size, const void* p) + { + fb_assert(blobStreamBuffer); + + const UCHAR* ptr = reinterpret_cast(p); + + while(size) + { + ULONG space = blobBufferSize - (blobStream - blobStreamBuffer); + ULONG step = MIN(size, space); + if (step == blobBufferSize) + { + // direct packet sent + sendBlobPacket(blobBufferSize, ptr); + } + else + { + // use buffer + memcpy(blobStream, ptr, step); + blobStream += step; + if (blobStream - blobStreamBuffer == blobBufferSize) + { + sendBlobPacket(blobBufferSize, blobStreamBuffer); + blobStream = blobStreamBuffer; + sizePointer = NULL; + } + } + + size -= step; + ptr += step; + } + } + + void setSizePointer() + { + fb_assert(FB_ALIGN(blobStream, sizeof(*sizePointer)) == blobStream); + sizePointer = reinterpret_cast(blobStream); + } + + void putSegment(ULONG size, const void* ptr) + { + if (!sizePointer) + { + newBlob(); + + ISC_QUAD zero = {0, 0}; + putBlobData(sizeof zero, &zero); + setSizePointer(); + ULONG z2 = 0; + putBlobData(sizeof z2, &z2); + putBlobData(sizeof z2, &z2); + } + + *sizePointer += size; + if (segmented) + { + if (size > MAX_USHORT) + { + (Arg::Gds(isc_imp_exc) << Arg::Gds(isc_blobtoobig) + << Arg::Gds(isc_random) << "Segment size >= 64Kb").raise(); + } + + alignBlobBuffer(BLOB_SEGHDR_ALIGN, sizePointer); + *sizePointer += sizeof(USHORT); + USHORT segSize = size; + putBlobData(sizeof segSize, &segSize); + } + putBlobData(size, ptr); + } + + void flashBatch() + { + alignBlobBuffer(blobAlign); + ULONG size = blobStream - blobStreamBuffer; + if (size) + { + sendBlobPacket(size, blobStreamBuffer); + blobStream = blobStreamBuffer; + } + + batchActive = false; + } + + void sendBlobPacket(unsigned size, const UCHAR* ptr); + void sendMessagePacket(unsigned size, const UCHAR* ptr); + + Firebird::AutoPtr > messageStreamBuffer, blobStreamBuffer; + ULONG messageStream; + UCHAR* blobStream; + ULONG* sizePointer; + + ULONG messageSize, alignedSize, blobBufferSize, messageBufferSize, flags; + Statement* stmt; + RefPtr format; + ISC_QUAD genId; + int blobAlign; + UCHAR blobPolicy; + bool segmented, defSegmented, batchActive; + +public: + bool tmpStatement; +}; + +int Batch::release() +{ + if (--refCounter != 0) + return 1; + + if (stmt) + { + LocalStatus ls; + CheckStatusWrapper status(&ls); + freeClientData(&status, true); + } + delete this; + + return 0; +} + class Statement FB_FINAL : public RefCntIface > { public: @@ -344,6 +556,9 @@ public: statement->rsr_timeout = timeOut; } + Batch* createBatch(CheckStatusWrapper* status, IMessageMetadata* inMetadata, + unsigned parLength, const unsigned char* par); + public: Statement(Rsr* handle, Attachment* a, unsigned aDialect) : metadata(getPool(), this, NULL), @@ -359,6 +574,11 @@ public: return statement; } + Attachment* getAttachment() + { + return remAtt; + } + void parseMetadata(const Array& buffer) { metadata.clear(); @@ -536,6 +756,10 @@ public: unsigned int getStatementTimeout(CheckStatusWrapper* status); void setStatementTimeout(CheckStatusWrapper* status, unsigned int timeOut); + Batch* createBatch(Firebird::CheckStatusWrapper* status, ITransaction* transaction, + unsigned stmtLength, const char* sqlStmt, unsigned dialect, + IMessageMetadata* inMetadata, unsigned parLength, const unsigned char* par); + public: Attachment(Rdb* handle, const PathName& path) : rdb(handle), dbPath(getPool(), path) @@ -1856,7 +2080,693 @@ void Attachment::setStatementTimeout(CheckStatusWrapper* status, unsigned int ti } -Firebird::ITransaction* Statement::execute(CheckStatusWrapper* status, Firebird::ITransaction* apiTra, +Batch* Attachment::createBatch(CheckStatusWrapper* status, ITransaction* transaction, + unsigned stmtLength, const char* sqlStmt, unsigned dialect, + IMessageMetadata* inMetadata, unsigned parLength, const unsigned char* par) +{ +/************************************** + * + * c r e a t e B a t c h + * + ************************************** + * + * Functional description + * Create jdbc-style batch for SQL statement. + * + **************************************/ + Statement* stmt = prepare(status, transaction, stmtLength, sqlStmt, dialect, 0); + if (status->getState() & Firebird::IStatus::STATE_ERRORS) + { + return NULL; + } + + Batch* rc = stmt->createBatch(status, inMetadata, parLength, par); + if (status->getState() & Firebird::IStatus::STATE_ERRORS) + { + stmt->release(); + return NULL; + } + + rc->tmpStatement = true; + return rc; +} + + +Batch* Statement::createBatch(CheckStatusWrapper* status, IMessageMetadata* inMetadata, + unsigned parLength, const unsigned char* par) +{ +/************************************** + * + * c r e a t e B a t c h + * + ************************************** + * + * Functional description + * Create jdbc-style batch for prepared statement. + * + **************************************/ + + try + { + reset(status); + + // Check and validate handles, etc. + CHECK_HANDLE(statement, isc_bad_req_handle); + Rdb* rdb = statement->rsr_rdb; + CHECK_HANDLE(rdb, isc_bad_db_handle); + rem_port* port = rdb->rdb_port; + + if (port->port_protocol < PROTOCOL_VERSION16) + unsupported(); + + // Build input BLR + RefPtr meta; + if (!inMetadata) + { + meta.assignRefNoIncr(getInputMetadata(status)); + check(status); + inMetadata = meta; + } + + BlrFromMessage inBlr(inMetadata, dialect, port->port_protocol); + const unsigned int in_blr_length = inBlr.getLength(); + const UCHAR* const in_blr = inBlr.getBytes(); + + // Validate data length + CHECK_LENGTH(port, in_blr_length); + + RefMutexGuard portGuard(*port->port_sync, FB_FUNCTION); + + delete statement->rsr_bind_format; + statement->rsr_bind_format = NULL; + + if (port->port_statement) + { + delete port->port_statement->rsr_select_format; + port->port_statement->rsr_select_format = NULL; + } + + // Parse the blr describing the message, if there is any. + if (in_blr_length) + statement->rsr_bind_format = PARSE_msg_format(in_blr, in_blr_length); + + RMessage* message = NULL; + if (!statement->rsr_buffer) + { + statement->rsr_buffer = message = FB_NEW RMessage(0); + statement->rsr_message = message; + + message->msg_next = message; + + statement->rsr_fmt_length = 0; + } + else + message = statement->rsr_message = statement->rsr_buffer; + + statement->rsr_flags.clear(Rsr::FETCHED); + statement->rsr_format = statement->rsr_bind_format; + statement->rsr_batch_stream.blobRemaining = 0; + statement->clearException(); + + // set up the packet for the other guy... + + PACKET* packet = &rdb->rdb_packet; + packet->p_operation = op_batch_create; + P_BATCH_CREATE* batch = &packet->p_batch_create; + batch->p_batch_statement = statement->rsr_id; + batch->p_batch_blr.cstr_length = in_blr_length; + batch->p_batch_blr.cstr_address = in_blr; + batch->p_batch_msglen = inMetadata->getMessageLength(status); + check(status); + batch->p_batch_pb.cstr_length = parLength; + batch->p_batch_pb.cstr_address = par; + + send_partial_packet(port, packet); + defer_packet(port, packet, true); + message->msg_address = NULL; + + Batch* b = FB_NEW Batch(this, inMetadata, parLength, par); + b->addRef(); + return b; + } + catch (const Exception& ex) + { + ex.stuffException(status); + } + + return NULL; +} + + +Batch::Batch(Statement* s, IMessageMetadata* inFmt, unsigned parLength, const unsigned char* par) + : messageStream(0), blobStream(nullptr), sizePointer(nullptr), + messageSize(0), alignedSize(0), blobBufferSize(0), messageBufferSize(0), flags(0), + stmt(s), format(inFmt), blobAlign(0), blobPolicy(BLOB_NONE), + segmented(false), defSegmented(false), batchActive(false), tmpStatement(false) +{ + LocalStatus ls; + CheckStatusWrapper st(&ls); + + messageSize = format->getMessageLength(&st); + check(&st); + alignedSize = format->getAlignedLength(&st); + check(&st); + + memset(&genId, 0, sizeof(genId)); + + ClumpletReader rdr(ClumpletReader::WideTagged, par, parLength); + + for (rdr.rewind(); !rdr.isEof(); rdr.moveNext()) + { + UCHAR t = rdr.getClumpTag(); + + switch (t) + { + case TAG_MULTIERROR: + case TAG_RECORD_COUNTS: + if (rdr.getInt()) + flags |= (1 << t); + else + flags &= ~(1 << t); + break; + + case TAG_BLOB_POLICY: + blobPolicy = rdr.getInt(); + + switch (blobPolicy) + { + case BLOB_ID_ENGINE: + case BLOB_ID_USER: + case BLOB_STREAM: + break; + default: + blobPolicy = BLOB_NONE; + break; + } + + break; + } + } + s->getStatement()->rsr_batch_flags = flags; + + // allocate buffers + Rsr* statement = stmt->getStatement(); + CHECK_HANDLE(statement, isc_bad_req_handle); + Rdb* rdb = statement->rsr_rdb; + CHECK_HANDLE(rdb, isc_bad_db_handle); + rem_port* port = rdb->rdb_port; + blobBufferSize = port->getPortConfig()->getClientBatchBuffer(); + messageBufferSize = blobBufferSize / alignedSize; + if (!messageBufferSize) + messageBufferSize = 1; + + messageStreamBuffer.reset(FB_NEW UCHAR[messageBufferSize * alignedSize]); + if (blobPolicy != BLOB_NONE) + { + blobStreamBuffer.reset(FB_NEW UCHAR[blobBufferSize]); + blobStream = blobStreamBuffer; + } +} + + +void Batch::add(CheckStatusWrapper* status, unsigned count, const void* inBuffer) +{ + try + { + // Check and validate handles, etc. + + if (!stmt) + { + Arg::Gds(isc_bad_req_handle).raise(); + } + + Rsr* statement = stmt->getStatement(); + CHECK_HANDLE(statement, isc_bad_req_handle); + Rdb* rdb = statement->rsr_rdb; + CHECK_HANDLE(rdb, isc_bad_db_handle); + rem_port* port = rdb->rdb_port; + + if (count == 0) + return; + + RefMutexGuard portGuard(*port->port_sync, FB_FUNCTION); + putMessageData(count, inBuffer); + + batchActive = true; + } + catch (const Exception& ex) + { + ex.stuffException(status); + } +} + + +void Batch::sendMessagePacket(unsigned count, const UCHAR* ptr) +{ + Rsr* statement = stmt->getStatement(); + CHECK_HANDLE(statement, isc_bad_req_handle); + Rdb* rdb = statement->rsr_rdb; + CHECK_HANDLE(rdb, isc_bad_db_handle); + rem_port* port = rdb->rdb_port; + + PACKET* packet = &rdb->rdb_packet; + packet->p_operation = op_batch_msg; + P_BATCH_MSG* batch = &packet->p_batch_msg; + batch->p_batch_statement = statement->rsr_id; + batch->p_batch_messages = count; + batch->p_batch_data.cstr_address = const_cast(ptr); + statement->rsr_batch_size = alignedSize; + + send_partial_packet(port, packet); + defer_packet(port, packet, true); +} + + +void Batch::addBlob(CheckStatusWrapper* status, unsigned length, const void* inBuffer, ISC_QUAD* blobId, + unsigned parLength, const unsigned char* par) +{ + try + { + // Check and validate handles, etc. + if (!stmt) + { + Arg::Gds(isc_bad_req_handle).raise(); + } + + Rsr* statement = stmt->getStatement(); + CHECK_HANDLE(statement, isc_bad_req_handle); + Rdb* rdb = statement->rsr_rdb; + CHECK_HANDLE(rdb, isc_bad_db_handle); + rem_port* port = rdb->rdb_port; + RefMutexGuard portGuard(*port->port_sync, FB_FUNCTION); + + // Policy check + switch(blobPolicy) + { + case IBatch::BLOB_ID_ENGINE: + genBlobId(blobId); + break; + case IBatch::BLOB_ID_USER: + break; + default: + (Arg::Gds(isc_random) << "Invalid blob policy for appendBlobData call").raise(); + } + + // Build blob HDR in stream + newBlob(); + putBlobData(sizeof *blobId, blobId); + setSizePointer(); + putBlobData(sizeof parLength, &parLength); + putBlobData(sizeof parLength, &parLength); + putBlobData(parLength, par); + segmented = parLength ? fb_utils::isBpbSegmented(parLength, par) : defSegmented; + + // Store blob data + putSegment(length, inBuffer); + + batchActive = true; + } + catch (const Exception& ex) + { + ex.stuffException(status); + } +} + + +void Batch::appendBlobData(CheckStatusWrapper* status, unsigned length, const void* inBuffer) +{ + try + { + // Check and validate handles, etc. + if (!stmt) + { + Arg::Gds(isc_bad_req_handle).raise(); + } + + // Policy check + switch(blobPolicy) + { + case IBatch::BLOB_ID_USER: + case IBatch::BLOB_ID_ENGINE: + break; + default: + (Arg::Gds(isc_random) << "Invalid blob policy for appendBlobData call").raise(); + } + + Rsr* statement = stmt->getStatement(); + CHECK_HANDLE(statement, isc_bad_req_handle); + Rdb* rdb = statement->rsr_rdb; + CHECK_HANDLE(rdb, isc_bad_db_handle); + rem_port* port = rdb->rdb_port; + RefMutexGuard portGuard(*port->port_sync, FB_FUNCTION); + + // Store blob data + putSegment(length, inBuffer); + } + catch (const Exception& ex) + { + ex.stuffException(status); + } +} + + +void Batch::addBlobStream(CheckStatusWrapper* status, unsigned length, const void* inBuffer) +{ + try + { + // Check and validate handles, etc. + if (!stmt) + { + Arg::Gds(isc_bad_req_handle).raise(); + } + + // Policy check + if (blobPolicy != IBatch::BLOB_STREAM) + { + (Arg::Gds(isc_random) << "Invalid blob policy for addBlobStream() call").raise(); + } + + Rsr* statement = stmt->getStatement(); + CHECK_HANDLE(statement, isc_bad_req_handle); + Rdb* rdb = statement->rsr_rdb; + CHECK_HANDLE(rdb, isc_bad_db_handle); + rem_port* port = rdb->rdb_port; + RefMutexGuard portGuard(*port->port_sync, FB_FUNCTION); + + // Store stream data + putBlobData(length, inBuffer); + + batchActive = true; + } + catch (const Exception& ex) + { + ex.stuffException(status); + } +} + + +void Batch::sendBlobPacket(unsigned size, const UCHAR* ptr) +{ + Rsr* statement = stmt->getStatement(); + Rdb* rdb = statement->rsr_rdb; + rem_port* port = rdb->rdb_port; + + setBlobAlignment(); + fb_assert(!(size % blobAlign)); + + PACKET* packet = &rdb->rdb_packet; + packet->p_operation = op_batch_blob_stream; + P_BATCH_BLOB* batch = &packet->p_batch_blob; + batch->p_batch_statement = statement->rsr_id; + batch->p_batch_blob_data.cstr_address = const_cast(ptr); + batch->p_batch_blob_data.cstr_length = size; + + send_partial_packet(port, packet); + defer_packet(port, packet, true); +} + + +void Batch::setDefaultBpb(CheckStatusWrapper* status, unsigned parLength, const unsigned char* par) +{ + try + { + // Check and validate handles, etc. + if (!stmt) + Arg::Gds(isc_bad_req_handle).raise(); + + Rsr* statement = stmt->getStatement(); + CHECK_HANDLE(statement, isc_bad_req_handle); + Rdb* rdb = statement->rsr_rdb; + CHECK_HANDLE(rdb, isc_bad_db_handle); + rem_port* port = rdb->rdb_port; + RefMutexGuard portGuard(*port->port_sync, FB_FUNCTION); + + // Check for presence of any data in batch buffers + if (batchHasData()) + (Arg::Gds(isc_random) << "Can't change default BPB after adding any data to batch").raise(); + + // Set default segmentation flag + defSegmented = fb_utils::isBpbSegmented(parLength, par); + + // Prepare and send the packet + PACKET* packet = &rdb->rdb_packet; + packet->p_operation = op_batch_set_bpb; + P_BATCH_SETBPB* batch = &packet->p_batch_setbpb; + batch->p_batch_statement = statement->rsr_id; + batch->p_batch_blob_bpb.cstr_address = par; + batch->p_batch_blob_bpb.cstr_length = parLength; + + send_partial_packet(port, packet); + defer_packet(port, packet, true); + } + catch (const Exception& ex) + { + ex.stuffException(status); + } +} + + +unsigned Batch::getBlobAlignment(CheckStatusWrapper* status) +{ + try + { + setBlobAlignment(); + } + catch (const Exception& ex) + { + ex.stuffException(status); + } + + return blobAlign; +} + + +void Batch::setBlobAlignment() +{ + if (blobAlign) + return; + + // Check and validate handles, etc. + if (!stmt) + { + Arg::Gds(isc_bad_req_handle).raise(); + } + + Rsr* statement = stmt->getStatement(); + CHECK_HANDLE(statement, isc_bad_req_handle); + Rdb* rdb = statement->rsr_rdb; + CHECK_HANDLE(rdb, isc_bad_db_handle); + rem_port* port = rdb->rdb_port; + RefMutexGuard portGuard(*port->port_sync, FB_FUNCTION); + + // Perform info call to server + LocalStatus ls; + CheckStatusWrapper s(&ls); + UCHAR item = isc_info_sql_stmt_blob_align; + UCHAR buffer[16]; + info(&s, rdb, op_info_sql, statement->rsr_id, 0, + 1, &item, 0, 0, sizeof(buffer), buffer); + check(&s); + + // Extract from buffer + if (buffer[0] != item) + (Arg::Gds(isc_random) << "Unexpected info buffer structure").raise(); + + int len = gds__vax_integer(&buffer[1], 2); + statement->rsr_batch_stream.alignment = blobAlign = gds__vax_integer(&buffer[3], len); +} + + +IMessageMetadata* Batch::getMetadata(CheckStatusWrapper* status) +{ + reset(status); + + format->addRef(); + return format; +} + + +void Batch::registerBlob(CheckStatusWrapper* status, const ISC_QUAD* existingBlob, ISC_QUAD* blobId) +{ + try + { + // Check and validate handles, etc. + + if (!stmt) + { + Arg::Gds(isc_bad_req_handle).raise(); + } + + Rsr* statement = stmt->getStatement(); + CHECK_HANDLE(statement, isc_bad_req_handle); + Rdb* rdb = statement->rsr_rdb; + CHECK_HANDLE(rdb, isc_bad_db_handle); + rem_port* port = rdb->rdb_port; + RefMutexGuard portGuard(*port->port_sync, FB_FUNCTION); + + if (blobPolicy == IBatch::BLOB_ID_ENGINE) + genBlobId(blobId); + + PACKET* packet = &rdb->rdb_packet; + packet->p_operation = op_batch_regblob; + P_BATCH_REGBLOB* batch = &packet->p_batch_regblob; + batch->p_batch_statement = statement->rsr_id; + batch->p_batch_exist_id = *existingBlob; + batch->p_batch_blob_id = *blobId; + + send_partial_packet(port, packet); + defer_packet(port, packet, true); + } + catch (const Exception& ex) + { + ex.stuffException(status); + } +} + + +IBatchCompletionState* Batch::execute(CheckStatusWrapper* status, ITransaction* apiTra) +{ + try + { + // Check and validate handles, etc. + + if (!stmt) + { + Arg::Gds(isc_bad_req_handle).raise(); + } + + Rsr* statement = stmt->getStatement(); + CHECK_HANDLE(statement, isc_bad_req_handle); + + Rdb* rdb = statement->rsr_rdb; + CHECK_HANDLE(rdb, isc_bad_db_handle); + + rem_port* port = rdb->rdb_port; + RefMutexGuard portGuard(*port->port_sync, FB_FUNCTION); + + Rtr* transaction = NULL; + Transaction* rt = stmt->getAttachment()->remoteTransactionInterface(apiTra); + if (rt) + { + transaction = rt->getTransaction(); + CHECK_HANDLE(transaction, isc_bad_trans_handle); + } + + // Sanity checks complete - flash data in buffers + flashBatch(); + + // Prepare and send execute packet + PACKET* packet = &rdb->rdb_packet; + packet->p_operation = op_batch_exec; + P_BATCH_EXEC* batch = &packet->p_batch_exec; + batch->p_batch_statement = statement->rsr_id; + batch->p_batch_transaction = transaction->rtr_id; + send_packet(port, packet); + + statement->rsr_batch_size = alignedSize; + AutoPtr > + cs(FB_NEW BatchCompletionState(flags & (1 << IBatch::TAG_RECORD_COUNTS), 256)); + statement->rsr_batch_cs = cs; + receive_packet(port, packet); + statement->rsr_batch_cs = nullptr; + + switch (packet->p_operation) + { + case op_response: + REMOTE_check_response(status, rdb, packet); + break; + + case op_batch_cs: + return cs.release(); + + default: + (Arg::Gds(isc_random) << "Unexpected packet type").raise(); + break; + } + } + catch (const Exception& ex) + { + ex.stuffException(status); + } + + return nullptr; +} + + +void Batch::cancel(CheckStatusWrapper* status) +{ + try + { + } + catch (const Exception& ex) + { + ex.stuffException(status); + } +} + + +void Batch::freeClientData(CheckStatusWrapper* status, bool force) +{ + try + { + // Check and validate handles, etc. + + if (!stmt) + { + Arg::Gds(isc_dsql_cursor_err).raise(); + } + + Rsr* statement = stmt->getStatement(); + CHECK_HANDLE(statement, isc_bad_req_handle); + Rdb* rdb = statement->rsr_rdb; + rem_port* port = rdb->rdb_port; + RefMutexGuard portGuard(*port->port_sync, FB_FUNCTION); + + PACKET* packet = &rdb->rdb_packet; + packet->p_operation = op_batch_rls; + + P_BATCH_FREE* batch = &packet->p_batch_free; + batch->p_batch_statement = statement->rsr_id; + + if (rdb->rdb_port->port_flags & PORT_lazy) + { + defer_packet(rdb->rdb_port, packet); + packet->p_resp.p_resp_object = statement->rsr_id; + } + else + { + try + { + send_and_receive(status, rdb, packet); + } + catch (const Exception&) + { + if (!force) + throw; + } + } + + releaseStatement(); + } + catch (const Exception& ex) + { + ex.stuffException(status); + } +} + + +void Batch::releaseStatement() +{ + if (tmpStatement) + { + stmt->release(); + } + + stmt = NULL; +} + + +ITransaction* Statement::execute(CheckStatusWrapper* status, ITransaction* apiTra, IMessageMetadata* inMetadata, void* inBuffer, IMessageMetadata* outMetadata, void* outBuffer) { /************************************** @@ -7321,7 +8231,8 @@ static void send_packet(rem_port* port, PACKET* packet) if (!p->sent) { if (!port->send_partial(&p->packet)) - Arg::Gds(isc_net_write_err).raise(); + (Arg::Gds(isc_net_write_err) << + Arg::Gds(isc_random) << "send_packet/send_partial").raise(); p->sent = true; } @@ -7330,7 +8241,7 @@ static void send_packet(rem_port* port, PACKET* packet) if (!port->send(packet)) { - Arg::Gds(isc_net_write_err).raise(); + (Arg::Gds(isc_net_write_err)<< Arg::Gds(isc_random) << "send_packet/send").raise(); } } @@ -7368,7 +8279,8 @@ static void send_partial_packet(rem_port* port, PACKET* packet) { if (!port->send_partial(&p->packet)) { - Arg::Gds(isc_net_write_err).raise(); + (Arg::Gds(isc_net_write_err) << + Arg::Gds(isc_random) << "send_partial_packet/send_partial").raise(); } p->sent = true; } @@ -7376,7 +8288,8 @@ static void send_partial_packet(rem_port* port, PACKET* packet) if (!port->send_partial(packet)) { - Arg::Gds(isc_net_write_err).raise(); + (Arg::Gds(isc_net_write_err) << + Arg::Gds(isc_random) << "send_partial_packet/send").raise(); } } diff --git a/src/remote/protocol.cpp b/src/remote/protocol.cpp index 4cc06be3e4..ee226ba00c 100644 --- a/src/remote/protocol.cpp +++ b/src/remote/protocol.cpp @@ -41,6 +41,9 @@ #include "../common/sdl_proto.h" #include "../common/StatusHolder.h" #include "../common/classes/stack.h" +#include "../common/classes/BatchCompletionState.h" +#include "../common/utils_proto.h" +#include "../dsql/DsqlBatch.h" using namespace Firebird; @@ -70,7 +73,7 @@ inline bool_t P_TRUE(XDR*, PACKET*) { return TRUE; } -inline bool_t P_FALSE(XDR*, PACKET*) +inline bool_t P_FALSE(XDR* xdrs, PACKET*) { return FALSE; } @@ -85,6 +88,8 @@ inline void DEBUG_XDR_FREE(XDR*, const void*, const void*, ULONG) } #endif // DEBUG_XDR_MEMORY +#define P_CHECK(xdr, p, st) if (st.getState() & IStatus::STATE_ERRORS) return P_FALSE(xdr, p) + #define MAP(routine, ptr) if (!routine (xdrs, &ptr)) return P_FALSE(xdrs, p); const ULONG MAX_OPAQUE = 32768; @@ -112,6 +117,10 @@ static bool_t xdr_sql_blr(XDR*, SLONG, CSTRING*, bool, SQL_STMT_TYPE); static bool_t xdr_sql_message(XDR*, SLONG); static bool_t xdr_trrq_blr(XDR*, CSTRING*); static bool_t xdr_trrq_message(XDR*, USHORT); +static bool_t xdr_bytes(XDR*, void*, ULONG); +static bool_t xdr_blob_stream(XDR*, SSHORT, CSTRING*); +static Rsr* getStatement(XDR*, USHORT); + #include "../common/xdr_proto.h" @@ -818,6 +827,281 @@ bool_t xdr_protocol(XDR* xdrs, PACKET* p) return P_TRUE(xdrs, p); } + case op_batch_create: + { + P_BATCH_CREATE* b = &p->p_batch_create; + MAP(xdr_short, reinterpret_cast(b->p_batch_statement)); + MAP(xdr_cstring_const, b->p_batch_blr); + MAP(xdr_u_long, b->p_batch_msglen); + MAP(xdr_cstring_const, b->p_batch_pb); + + DEBUG_PRINTSIZE(xdrs, p->p_operation); + + return P_TRUE(xdrs, p); + } + + case op_batch_msg: + { + P_BATCH_MSG* b = &p->p_batch_msg; + MAP(xdr_short, reinterpret_cast(b->p_batch_statement)); + MAP(xdr_u_long, b->p_batch_messages); + + if (xdrs->x_op == XDR_FREE) + { + MAP(xdr_cstring, b->p_batch_data); + return P_TRUE(xdrs, p); + } + + rem_port* port = (rem_port*) xdrs->x_public; + SSHORT statement_id = b->p_batch_statement; + Rsr* statement; + if (statement_id >= 0) + { + if (static_cast(statement_id) >= port->port_objects.getCount()) + return P_FALSE(xdrs, p); + + try + { + statement = port->port_objects[statement_id]; + } + catch (const status_exception&) + { + return P_FALSE(xdrs, p); + } + } + else + { + statement = port->port_statement; + } + + if (!statement) + return P_FALSE(xdrs, p); + + ULONG count = b->p_batch_messages; + ULONG size = statement->rsr_batch_size; + if (xdrs->x_op == XDR_DECODE) + { + b->p_batch_data.cstr_length = (count ? count : 1) * size; + alloc_cstring(xdrs, &b->p_batch_data); + } + + RMessage* message = statement->rsr_buffer; + if (!message) + return P_FALSE(xdrs, p); + statement->rsr_buffer = message->msg_next; + message->msg_address = b->p_batch_data.cstr_address; + + while (count--) + { + DEB_BATCH(fprintf(stderr, "BatRem: xdr packed msg\n")); + if (!xdr_packed_message(xdrs, message, statement->rsr_format)) + return P_FALSE(xdrs, p); + message->msg_address += statement->rsr_batch_size; + } + + message->msg_address = nullptr; + DEBUG_PRINTSIZE(xdrs, p->p_operation); + + return P_TRUE(xdrs, p); + } + + case op_batch_exec: + { + P_BATCH_EXEC* b = &p->p_batch_exec; + MAP(xdr_short, reinterpret_cast(b->p_batch_statement)); + MAP(xdr_short, reinterpret_cast(b->p_batch_transaction)); + + if (xdrs->x_op != XDR_FREE) + DEB_BATCH(fprintf(stderr, "BatRem: xdr execute\n")); + + return P_TRUE(xdrs, p); + } + + case op_batch_cs: + { + P_BATCH_CS* b = &p->p_batch_cs; + MAP(xdr_short, reinterpret_cast(b->p_batch_statement)); + MAP(xdr_u_long, b->p_batch_reccount); + MAP(xdr_u_long, b->p_batch_updates); + MAP(xdr_u_long, b->p_batch_vectors); + MAP(xdr_u_long, b->p_batch_errors); + + if (xdrs->x_op == XDR_FREE) + return P_TRUE(xdrs, p); + + rem_port* port = (rem_port*) xdrs->x_public; + SSHORT statement_id = b->p_batch_statement; + DEB_BATCH(fprintf(stderr, "BatRem: xdr CS %d\n", statement_id)); + Rsr* statement; + + if (statement_id >= 0) + { + if (static_cast(statement_id) >= port->port_objects.getCount()) + return P_FALSE(xdrs, p); + + try + { + statement = port->port_objects[statement_id]; + } + catch (const status_exception&) + { + return P_FALSE(xdrs, p); + } + } + else + { + statement = port->port_statement; + } + + if (!statement) + return P_FALSE(xdrs, p); + + LocalStatus ls; + CheckStatusWrapper status_vector(&ls); + + if ((xdrs->x_op == XDR_DECODE) && (!b->p_batch_updates)) + { + DEB_BATCH(fprintf(stderr, "BatRem: xdr reccount=%d\n", b->p_batch_reccount)); + statement->rsr_batch_cs->regSize(b->p_batch_reccount); + } + + // Process update counters + DEB_BATCH(fprintf(stderr, "BatRem: xdr up %d\n", b->p_batch_updates)); + for (unsigned i = 0; i < b->p_batch_updates; ++i) + { + SLONG v; + + if (xdrs->x_op == XDR_ENCODE) + { + v = statement->rsr_batch_ics->getState(&status_vector, i); + P_CHECK(xdrs, p, status_vector); + } + + MAP(xdr_long, v); + + if (xdrs->x_op == XDR_DECODE) + { + statement->rsr_batch_cs->regUpdate(v); + } + } + + // Process status vectors + ULONG pos = 0u; + LocalStatus to; + DEB_BATCH(fprintf(stderr, "BatRem: xdr sv %d\n", b->p_batch_vectors)); + + for (unsigned i = 0; i < b->p_batch_vectors; ++i, ++pos) + { + DynamicStatusVector s; + DynamicStatusVector* ptr = NULL; + + if (xdrs->x_op == XDR_ENCODE) + { + pos = statement->rsr_batch_ics->findError(&status_vector, pos); + P_CHECK(xdrs, p, status_vector); + if (pos == IBatchCompletionState::NO_MORE_ERRORS) + return P_FALSE(xdrs, p); + + statement->rsr_batch_ics->getStatus(&status_vector, &to, pos); + if (status_vector.getState() & IStatus::STATE_ERRORS) + continue; + + s.load(&to); + ptr = &s; + } + + MAP(xdr_u_long, pos); + + if (!xdr_status_vector(xdrs, ptr)) + return P_FALSE(xdrs, p); + + if (xdrs->x_op == XDR_DECODE) + { + Firebird::Arg::StatusVector sv(ptr->value()); + sv.copyTo(&to); + delete ptr; + statement->rsr_batch_cs->regErrorAt(pos, &to); + } + } + + // Process status-less errors + pos = 0u; + DEB_BATCH(fprintf(stderr, "BatRem: xdr err %d\n", b->p_batch_errors)); + + for (unsigned i = 0; i < b->p_batch_errors; ++i, ++pos) + { + if (xdrs->x_op == XDR_ENCODE) + { + pos = statement->rsr_batch_ics->findError(&status_vector, pos); + P_CHECK(xdrs, p, status_vector); + if (pos == IBatchCompletionState::NO_MORE_ERRORS) + return P_FALSE(xdrs, p); + + statement->rsr_batch_ics->getStatus(&status_vector, &to, pos); + if (!(status_vector.getState() & IStatus::STATE_ERRORS)) + continue; + } + + MAP(xdr_u_long, pos); + + if (xdrs->x_op == XDR_DECODE) + { + statement->rsr_batch_cs->regErrorAt(pos, nullptr); + } + } + + return P_TRUE(xdrs, p); + } + + case op_batch_rls: + { + P_BATCH_FREE* b = &p->p_batch_free; + MAP(xdr_short, reinterpret_cast(b->p_batch_statement)); + + if (xdrs->x_op != XDR_FREE) + DEB_BATCH(fprintf(stderr, "BatRem: xdr release\n")); + + return P_TRUE(xdrs, p); + } + + case op_batch_set_bpb: + { + P_BATCH_SETBPB* b = &p->p_batch_setbpb; + MAP(xdr_short, reinterpret_cast(b->p_batch_statement)); + MAP(xdr_cstring_const, b->p_batch_blob_bpb); + + Rsr* statement = getStatement(xdrs, b->p_batch_statement); + if (!statement) + return P_FALSE(xdrs, p); + if (fb_utils::isBpbSegmented(b->p_batch_blob_bpb.cstr_length, b->p_batch_blob_bpb.cstr_address)) + statement->rsr_batch_flags |= (1 << Jrd::DsqlBatch::FLAG_DEFAULT_SEGMENTED); + else + statement->rsr_batch_flags &= ~(1 << Jrd::DsqlBatch::FLAG_DEFAULT_SEGMENTED); + + return P_TRUE(xdrs, p); + } + + case op_batch_regblob: + { + P_BATCH_REGBLOB* b = &p->p_batch_regblob; + MAP(xdr_short, reinterpret_cast(b->p_batch_statement)); + MAP(xdr_quad, b->p_batch_exist_id); + MAP(xdr_quad, b->p_batch_blob_id); + + return P_TRUE(xdrs, p); + } + + case op_batch_blob_stream: + { + P_BATCH_BLOB* b = &p->p_batch_blob; + MAP(xdr_short, reinterpret_cast(b->p_batch_statement)); + if (!xdr_blob_stream(xdrs, b->p_batch_statement, &b->p_batch_blob_data)) + return P_FALSE(xdrs, p); + + return P_TRUE(xdrs, p); + } + + ///case op_insert: default: #ifdef DEV_BUILD @@ -831,6 +1115,25 @@ bool_t xdr_protocol(XDR* xdrs, PACKET* p) } +static bool_t xdr_bytes(XDR* xdrs, void* bytes, ULONG size) +{ + switch (xdrs->x_op) + { + case XDR_ENCODE: + if (!xdrs->x_ops->x_putbytes(xdrs, reinterpret_cast(bytes), size)) + return FALSE; + break; + + case XDR_DECODE: + if (!xdrs->x_ops->x_getbytes(xdrs, reinterpret_cast(bytes), size)) + return FALSE; + break; + } + + return TRUE; +} + + ULONG xdr_protocol_overhead(P_OP op) { /************************************** @@ -1888,3 +2191,245 @@ static void reset_statement( XDR* xdrs, SSHORT statement_id) {} // no-op } } + +static Rsr* getStatement(XDR* xdrs, USHORT statement_id) +{ + rem_port* port = (rem_port*) xdrs->x_public; + + if (statement_id >= 0) + { + if (statement_id >= port->port_objects.getCount()) + return nullptr; + + try + { + return port->port_objects[statement_id]; + } + catch (const status_exception&) + { + return nullptr; + } + } + + return port->port_statement; +} + +static bool_t xdr_blob_stream(XDR* xdrs, SSHORT statement_id, CSTRING* strmPortion) +{ + if (xdrs->x_op == XDR_FREE) + return xdr_cstring(xdrs, strmPortion); + + Rsr* statement = getStatement(xdrs, statement_id); + if (!statement) + return FALSE; + + // create local copy - required in a case when packet is not complete and will be restarted + Rsr::BatchStream localStrm(statement->rsr_batch_stream); + + struct BlobFlow + { + ULONG remains; + UCHAR* streamPtr; + ULONG& blobSize; + ULONG& bpbSize; + ULONG& segSize; + + BlobFlow(Rsr::BatchStream* bs) + : remains(0), streamPtr(NULL), + blobSize(bs->blobRemaining), bpbSize(bs->bpbRemaining), segSize(bs->segRemaining) + { } + + void newBlob(ULONG totalSize, ULONG parSize) + { + blobSize = totalSize; + bpbSize = parSize; + segSize = 0; + } + + void move(ULONG step) + { + move2(step); + blobSize -= step; + } + + void moveBpb(ULONG step) + { + move(step); + bpbSize -= step; + } + + void moveSeg(ULONG step) + { + move(step); + segSize -= step; + } + + bool align(ULONG alignment) + { + ULONG a = IPTR(streamPtr) % alignment; + if (a) + { + a = alignment - a; + move2(a); + if (blobSize) + blobSize -= a; + } + return a; + } + +private: + void move2(ULONG step) + { + streamPtr += step; + remains -= step; + } + }; + + BlobFlow flow(&localStrm); + + if (xdrs->x_op == XDR_ENCODE) + { + flow.remains = strmPortion->cstr_length; + strmPortion->cstr_length += localStrm.hdrPrevious; + } + if (!xdr_u_long(xdrs, &strmPortion->cstr_length)) + return FALSE; + if (xdrs->x_op == XDR_DECODE) + flow.remains = strmPortion->cstr_length; + + fb_assert(localStrm.alignment); + if (flow.remains % localStrm.alignment) + return FALSE; + if (!flow.remains) + return TRUE; + + if (xdrs->x_op == XDR_DECODE) + alloc_cstring(xdrs, strmPortion); + + flow.streamPtr = strmPortion->cstr_address; + if (IPTR(flow.streamPtr) % localStrm.alignment != 0) + return FALSE; + + while (flow.remains) + { + if (!flow.blobSize) // we should process next blob header + { + // align data stream + if (flow.align(localStrm.alignment)) + continue; + + // check for partial header in the stream + if (flow.remains + localStrm.hdrPrevious < Rsr::BatchStream::SIZEOF_BLOB_HEAD) + { + // On the receiver that means packet protocol processing is complete: actual + // size of packet is sligtly less than passed in batch_blob_data.cstr_length. + if (xdrs->x_op == XDR_DECODE) + strmPortion->cstr_length -= flow.remains; + // On transmitter reserve partial header for future use + else + localStrm.saveData(flow.streamPtr, flow.remains); + + // Done with packet + break; + } + + // parse blob header + fb_assert(intptr_t(flow.streamPtr) % localStrm.alignment == 0); + unsigned char* hdrPtr = flow.streamPtr; // default is to use header in main buffer + unsigned hdrOffset = Rsr::BatchStream::SIZEOF_BLOB_HEAD; + if (localStrm.hdrPrevious) + { + // on transmitter reserved partial header may be used + fb_assert(xdrs->x_op == XDR_ENCODE); + hdrOffset -= localStrm.hdrPrevious; + localStrm.saveData(flow.streamPtr, hdrOffset); + hdrPtr = localStrm.hdr; + } + + ISC_QUAD* batchBlobId = reinterpret_cast(hdrPtr); + ULONG* blobSize = reinterpret_cast(hdrPtr + sizeof(ISC_QUAD)); + ULONG* bpbSize = reinterpret_cast(hdrPtr + sizeof(ISC_QUAD) + sizeof(ULONG)); + if (!xdr_quad(xdrs, batchBlobId)) + return FALSE; + if (!xdr_u_long(xdrs, blobSize)) + return FALSE; + if (!xdr_u_long(xdrs, bpbSize)) + return FALSE; + + flow.move(hdrOffset); + localStrm.hdrPrevious = 0; + flow.newBlob(*blobSize, *bpbSize); + localStrm.curBpb.clear(); + + if (!flow.bpbSize) + localStrm.segmented = statement->rsr_batch_flags & (1 << Jrd::DsqlBatch::FLAG_DEFAULT_SEGMENTED); + + continue; + } + + // process BPB + if (flow.bpbSize) + { + ULONG size = MIN(flow.bpbSize, flow.remains); + if (!xdr_bytes(xdrs, flow.streamPtr, size)) + return FALSE; + localStrm.curBpb.add(flow.streamPtr, size); + flow.moveBpb(size); + if (flow.bpbSize == 0) // bpb is passed completely + { + try + { + localStrm.segmented = fb_utils::isBpbSegmented(localStrm.curBpb.getCount(), + localStrm.curBpb.begin()); + } + catch (const Exception&) + { + return FALSE; + } + localStrm.curBpb.clear(); + } + + continue; + } + + // pass data + ULONG dataSize = MIN(flow.blobSize, flow.remains); + if (dataSize) + { + if (localStrm.segmented) + { + if (!flow.segSize) + { + if (flow.align(IBatch::BLOB_SEGHDR_ALIGN)) + continue; + + USHORT* segSize = reinterpret_cast(flow.streamPtr); + if (!xdr_u_short(xdrs, segSize)) + return FALSE; + flow.segSize = *segSize; + flow.move(sizeof(USHORT)); + + if (flow.segSize > flow.blobSize) + return FALSE; + } + + dataSize = MIN(flow.segSize, flow.remains); + if (!xdr_bytes(xdrs, flow.streamPtr, dataSize)) + return FALSE; + flow.moveSeg(dataSize); + } + else + { + if (!xdr_bytes(xdrs, flow.streamPtr, dataSize)) + return FALSE; + flow.move(dataSize); + } + } + } + + // packet processed successfully - save stream data for next one + statement->rsr_batch_stream = localStrm; + + return TRUE; +} + diff --git a/src/remote/protocol.h b/src/remote/protocol.h index c33ff0a818..0e6403aa5b 100644 --- a/src/remote/protocol.h +++ b/src/remote/protocol.h @@ -276,6 +276,15 @@ enum P_OP op_cond_accept = 98, // Server accepts connection, returns some data to client // and asks client to continue authentication before attach call + op_batch_create = 99, + op_batch_msg = 100, + op_batch_exec = 101, + op_batch_rls = 102, + op_batch_cs = 103, + op_batch_regblob = 104, + op_batch_blob_stream = 105, + op_batch_set_bpb = 106, + op_max }; @@ -645,6 +654,63 @@ typedef struct p_crypt_callback } P_CRYPT_CALLBACK; +// Batch definitions + +typedef struct p_batch_create +{ + OBJCT p_batch_statement; // statement object + CSTRING_CONST p_batch_blr; // blr describing input messages + ULONG p_batch_msglen; // explicit message length + CSTRING_CONST p_batch_pb; // parameters block +} P_BATCH_CREATE; + +typedef struct p_batch_msg +{ + OBJCT p_batch_statement; // statement object + ULONG p_batch_messages; // number of messages + CSTRING p_batch_data; +} P_BATCH_MSG; + +typedef struct p_batch_exec +{ + OBJCT p_batch_statement; // statement object + OBJCT p_batch_transaction; // transaction object +} P_BATCH_EXEC; + +typedef struct p_batch_cs // completion state +{ + OBJCT p_batch_statement; // statement object + ULONG p_batch_reccount; // total records + ULONG p_batch_updates; // update counters + ULONG p_batch_vectors; // recnum + status vector pairs + ULONG p_batch_errors; // error's recnums +} P_BATCH_CS; + +typedef struct p_batch_free +{ + OBJCT p_batch_statement; // statement object +} P_BATCH_FREE; + +typedef struct p_batch_blob +{ + OBJCT p_batch_statement; // statement object + CSTRING p_batch_blob_data; // data +} P_BATCH_BLOB; + +typedef struct p_batch_regblob +{ + OBJCT p_batch_statement; // statement object + SQUAD p_batch_exist_id; // id of blob to register + SQUAD p_batch_blob_id; // blob id +} P_BATCH_REGBLOB; + +typedef struct p_batch_setbpb +{ + OBJCT p_batch_statement; // statement object + CSTRING_CONST p_batch_blob_bpb; // BPB +} P_BATCH_SETBPB; + + // Generalize packet (sic!) typedef struct packet @@ -688,6 +754,14 @@ typedef struct packet P_AUTH_CONT p_auth_cont; // Request more auth data P_CRYPT p_crypt; // Start wire crypt P_CRYPT_CALLBACK p_cc; // Database crypt callback + P_BATCH_CREATE p_batch_create; // Create batch interface + P_BATCH_MSG p_batch_msg; // Add messages to batch + P_BATCH_EXEC p_batch_exec; // Run batch + P_BATCH_FREE p_batch_free; // Destroy batch + P_BATCH_CS p_batch_cs; // Batch completion state + P_BATCH_BLOB p_batch_blob; // BLOB stream portion in batch + P_BATCH_REGBLOB p_batch_regblob; // Register already existing BLOB in batch + P_BATCH_SETBPB p_batch_setbpb; // Set default BPB for batch public: packet() diff --git a/src/remote/remote.h b/src/remote/remote.h index 5e188b2e2a..26c1404d3d 100644 --- a/src/remote/remote.h +++ b/src/remote/remote.h @@ -64,6 +64,8 @@ //#define COMPRESS_DEBUG 1 #endif // WIRE_COMPRESS_SUPPORT +#define DEB_BATCH(x) + #define REM_SEND_OFFSET(bs) (0) #define REM_RECV_OFFSET(bs) (bs) @@ -103,6 +105,7 @@ const ULONG MAX_BATCH_CACHE_SIZE = 1024 * 1024; // 1 MB // fwd. decl. namespace Firebird { class Exception; + class BatchCompletionState; } #ifdef WIN_NT @@ -127,6 +130,7 @@ typedef Firebird::RefPtr ServBlob; typedef Firebird::RefPtr ServTransaction; typedef Firebird::RefPtr ServStatement; typedef Firebird::RefPtr ServCursor; +typedef Firebird::RefPtr ServBatch; typedef Firebird::RefPtr ServRequest; typedef Firebird::RefPtr ServEvents; typedef Firebird::RefPtr ServService; @@ -464,6 +468,7 @@ struct Rsr : public Firebird::GlobalStorage, public TypedHandle Rtr* rsr_rtr; ServStatement rsr_iface; ServCursor rsr_cursor; + ServBatch rsr_batch; rem_fmt* rsr_bind_format; // Format of bind message rem_fmt* rsr_select_format; // Format of select message rem_fmt* rsr_user_select_format; // Format of user's select message @@ -485,6 +490,41 @@ struct Rsr : public Firebird::GlobalStorage, public TypedHandle unsigned int rsr_timeout; // Statement timeout to be set on open\execute Rsr** rsr_self; + ULONG rsr_batch_size; // Aligned message size for IBatch operations + ULONG rsr_batch_flags; // Flags for batch processing + union // BatchCS passed to XDR protocol + { + Firebird::IBatchCompletionState* rsr_batch_ics; // server + Firebird::BatchCompletionState* rsr_batch_cs; // client + }; + + struct BatchStream + { + BatchStream() + : curBpb(*getDefaultMemoryPool()), hdrPrevious(0), segmented(false) + { } + + static const ULONG SIZEOF_BLOB_HEAD = sizeof(ISC_QUAD) + 2 * sizeof(ULONG); + + typedef Firebird::HalfStaticArray Bpb; + Bpb curBpb; + UCHAR hdr[SIZEOF_BLOB_HEAD]; + ULONG blobRemaining; // Remaining to transfer size of blob data + ULONG bpbRemaining; // Remaining to transfer size of BPB + ULONG segRemaining; // Remaining to transfer size of segment data + USHORT alignment; // Alignment in BLOB stream + USHORT hdrPrevious; // Header data left from previous block (in hdr) + bool segmented; // Current blob kind + + void saveData(const UCHAR* data, ULONG size) + { + fb_assert(size + hdrPrevious <= SIZEOF_BLOB_HEAD); + memcpy(&hdr[hdrPrevious], data, size); + hdrPrevious += size; + } + }; + BatchStream rsr_batch_stream; + public: // Values for rsr_flags. enum { @@ -500,7 +540,7 @@ public: public: Rsr() : - rsr_next(0), rsr_rdb(0), rsr_rtr(0), rsr_iface(NULL), rsr_cursor(NULL), + rsr_next(0), rsr_rdb(0), rsr_rtr(0), rsr_iface(NULL), rsr_cursor(NULL), rsr_batch(NULL), rsr_bind_format(0), rsr_select_format(0), rsr_user_select_format(0), rsr_format(0), rsr_message(0), rsr_buffer(0), rsr_status(0), rsr_id(0), rsr_fmt_length(0), @@ -532,6 +572,7 @@ public: static ISC_STATUS badHandle() { return isc_bad_req_handle; } void checkIface(ISC_STATUS code = isc_unprepared_stmt); void checkCursor(); + void checkBatch(); }; @@ -1211,6 +1252,13 @@ public: ISC_STATUS transact_request(P_TRRQ *, PACKET*); SSHORT asyncReceive(PACKET* asyncPacket, const UCHAR* buffer, SSHORT dataSize); void start_crypt(P_CRYPT*, PACKET*); + void batch_create(P_BATCH_CREATE*, PACKET*); + void batch_msg(P_BATCH_MSG*, PACKET*); + void batch_blob_stream(P_BATCH_BLOB*, PACKET*); + void batch_regblob(P_BATCH_REGBLOB*, PACKET*); + void batch_exec(P_BATCH_EXEC*, PACKET*); + void batch_rls(P_BATCH_FREE*, PACKET*); + void batch_bpb(P_BATCH_SETBPB*, PACKET*); Firebird::string getRemoteId() const; void auxAcceptError(PACKET* packet); diff --git a/src/remote/server/server.cpp b/src/remote/server/server.cpp index 926753786b..4d64a866e3 100644 --- a/src/remote/server/server.cpp +++ b/src/remote/server/server.cpp @@ -2059,6 +2059,13 @@ void Rsr::checkCursor() } +void Rsr::checkBatch() +{ + if (!rsr_batch) + (Arg::Gds(isc_random) << "Batch is not created").raise(); +} + + static ISC_STATUS allocate_statement( rem_port* port, /*P_RLSE* allocate,*/ PACKET* send) { /************************************** @@ -3318,6 +3325,231 @@ ISC_STATUS rem_port::execute_immediate(P_OP op, P_SQLST * exnow, PACKET* sendL) } +void rem_port::batch_create(P_BATCH_CREATE* batch, PACKET* sendL) +{ + LocalStatus ls; + CheckStatusWrapper status_vector(&ls); + Rsr* statement; + getHandle(statement, batch->p_batch_statement); + statement->checkIface(); + + const ULONG blr_length = batch->p_batch_blr.cstr_length; + const UCHAR* blr = batch->p_batch_blr.cstr_address; + if (!blr) + (Arg::Gds(isc_random) << "Missing required format info in createBatch()").raise(); + InternalMessageBuffer msgBuffer(blr_length, blr, batch->p_batch_msglen, NULL); + + // Flush out any previous format information + // that might be hanging around from an earlier execution. + + delete statement->rsr_bind_format; + statement->rsr_bind_format = PARSE_msg_format(blr, blr_length); + + // If we know the length of the message, make sure there is a buffer + // large enough to hold it. + + if (!(statement->rsr_format = statement->rsr_bind_format)) + (Arg::Gds(isc_random) << "Error parsing message format in createBatch()").raise(); + + RMessage* message = statement->rsr_buffer; + if (!message || statement->rsr_format->fmt_length > statement->rsr_fmt_length) + { + RMessage* const org_message = message; + const ULONG org_length = message ? statement->rsr_fmt_length : 0; + statement->rsr_fmt_length = statement->rsr_format->fmt_length; + statement->rsr_buffer = message = FB_NEW RMessage(statement->rsr_fmt_length); + statement->rsr_message = message; + message->msg_next = message; + if (org_length) + { + // dimitr: the original buffer might have something useful inside + // (filled by a prior xdr_sql_message() call, for example), + // so its contents must be preserved (see CORE-3730) + memcpy(message->msg_buffer, org_message->msg_buffer, org_length); + } + REMOTE_release_messages(org_message); + } + + ClumpletWriter wrt(ClumpletReader::WideTagged, MAX_DPB_SIZE, + batch->p_batch_pb.cstr_address, batch->p_batch_pb.cstr_length); + if (wrt.getBufferLength() && (wrt.getBufferTag() != IBatch::VERSION1)) + (Arg::Gds(isc_random) << "Invalid tag in parameters block").raise(); + statement->rsr_batch_flags = (wrt.find(IBatch::TAG_RECORD_COUNTS) && wrt.getInt()) ? + (1 << IBatch::TAG_RECORD_COUNTS) : 0; + if (wrt.find(IBatch::TAG_BLOB_POLICY) && (wrt.getInt() != IBatch::BLOB_STREAM)) + { + // we always send blobs in a stream therefore change policy + wrt.deleteClumplet(); + wrt.insertInt(IBatch::TAG_BLOB_POLICY, IBatch::BLOB_STREAM); + } + + statement->rsr_batch = + statement->rsr_iface->createBatch(&status_vector, msgBuffer.metadata, + wrt.getBufferLength(), wrt.getBuffer()); + + statement->rsr_batch_size = 0; + statement->rsr_batch_stream.blobRemaining = 0; + if (!(status_vector.getState() & Firebird::IStatus::STATE_ERRORS)) + { + if (msgBuffer.metadata) + { + statement->rsr_batch_size = msgBuffer.metadata->getAlignedLength(&status_vector); + check(&status_vector); + } + else + { + IMessageMetadata* m = statement->rsr_iface->getInputMetadata(&status_vector); + check(&status_vector); + statement->rsr_batch_size = m->getAlignedLength(&status_vector); + m->release(); + check(&status_vector); + } + + statement->rsr_batch_stream.alignment = statement->rsr_batch->getBlobAlignment(&status_vector); + check(&status_vector); + } + + this->send_response(sendL, 0, 0, &status_vector, true); +} + +void rem_port::batch_msg(P_BATCH_MSG* batch, PACKET* sendL) +{ + LocalStatus ls; + CheckStatusWrapper status_vector(&ls); + + Rsr* statement; + getHandle(statement, batch->p_batch_statement); + statement->checkIface(); + statement->checkBatch(); + + const ULONG count = batch->p_batch_messages; + const void* data = batch->p_batch_data.cstr_address; + + statement->rsr_batch->add(&status_vector, count, data); + + this->send_response(sendL, 0, 0, &status_vector, true); +} + + +void rem_port::batch_blob_stream(P_BATCH_BLOB* batch, PACKET* sendL) +{ + LocalStatus ls; + CheckStatusWrapper status_vector(&ls); + + Rsr* statement; + getHandle(statement, batch->p_batch_statement); + statement->checkIface(); + statement->checkBatch(); + + statement->rsr_batch->addBlobStream(&status_vector, + batch->p_batch_blob_data.cstr_length, batch->p_batch_blob_data.cstr_address); + + this->send_response(sendL, 0, 0, &status_vector, true); +} + +void rem_port::batch_bpb(P_BATCH_SETBPB* batch, PACKET* sendL) +{ + LocalStatus ls; + CheckStatusWrapper status_vector(&ls); + + Rsr* statement; + getHandle(statement, batch->p_batch_statement); + statement->checkIface(); + statement->checkBatch(); + + statement->rsr_batch->setDefaultBpb(&status_vector, + batch->p_batch_blob_bpb.cstr_length, batch->p_batch_blob_bpb.cstr_address); + + this->send_response(sendL, 0, 0, &status_vector, true); +} + + +void rem_port::batch_regblob(P_BATCH_REGBLOB* batch, PACKET* sendL) +{ + LocalStatus ls; + CheckStatusWrapper status_vector(&ls); + + Rsr* statement; + getHandle(statement, batch->p_batch_statement); + statement->checkIface(); + statement->checkBatch(); + + statement->rsr_batch->registerBlob(&status_vector, &batch->p_batch_exist_id, + &batch->p_batch_blob_id); + + this->send_response(sendL, 0, 0, &status_vector, true); +} + + +void rem_port::batch_exec(P_BATCH_EXEC* batch, PACKET* sendL) +{ + LocalStatus ls; + CheckStatusWrapper status_vector(&ls); + + Rsr* statement; + getHandle(statement, batch->p_batch_statement); + statement->checkIface(); + statement->checkBatch(); + + Rtr* transaction = NULL; + getHandle(transaction, batch->p_batch_transaction); + + AutoPtr > + ics(statement->rsr_batch->execute(&status_vector, transaction->rtr_iface)); + + if (status_vector.getState() & IStatus::STATE_ERRORS) + { + this->send_response(sendL, 0, 0, &status_vector, false); + return; + } + + bool recordCounts = statement->rsr_batch_flags & (1 << IBatch::TAG_RECORD_COUNTS); + P_BATCH_CS* pcs = &sendL->p_batch_cs; + sendL->p_operation = op_batch_cs; + pcs->p_batch_statement = statement->rsr_id; + pcs->p_batch_reccount = ics->getSize(&status_vector); + check(&status_vector); + pcs->p_batch_updates = recordCounts ? pcs->p_batch_reccount : 0; + pcs->p_batch_errors = pcs->p_batch_vectors = 0; + + for (unsigned int pos = 0u; + (pos = ics->findError(&status_vector, pos)) != IBatchCompletionState::NO_MORE_ERRORS; + ++pos) + { + check(&status_vector); + + LocalStatus dummy; + ics->getStatus(&status_vector, &dummy, pos); + if (status_vector.getState() & IStatus::STATE_ERRORS) + pcs->p_batch_errors++; + else + pcs->p_batch_vectors++; + } + + check(&status_vector); + + statement->rsr_batch_ics = ics; + this->send(sendL); +} + + +void rem_port::batch_rls(P_BATCH_FREE* batch, PACKET* sendL) +{ + LocalStatus ls; + CheckStatusWrapper status_vector(&ls); + + Rsr* statement; + getHandle(statement, batch->p_batch_statement); + statement->checkIface(); + statement->checkBatch(); + + statement->rsr_batch->release(); + statement->rsr_batch = nullptr; + + this->send_response(sendL, 0, 0, &status_vector, true); +} + + ISC_STATUS rem_port::execute_statement(P_OP op, P_SQLDATA* sqldata, PACKET* sendL) { /***************************************** @@ -4533,6 +4765,33 @@ static bool process_packet(rem_port* port, PACKET* sendL, PACKET* receive, rem_p port->start_crypt(&receive->p_crypt, sendL); break; + case op_batch_create: + port->batch_create(&receive->p_batch_create, sendL); + break; + + case op_batch_msg: + port->batch_msg(&receive->p_batch_msg, sendL); + break; + + case op_batch_exec: + port->batch_exec(&receive->p_batch_exec, sendL); + break; + + case op_batch_rls: + port->batch_rls(&receive->p_batch_free, sendL); + break; + + case op_batch_blob_stream: + port->batch_blob_stream(&receive->p_batch_blob, sendL); + break; + + case op_batch_regblob: + port->batch_regblob(&receive->p_batch_regblob, sendL); + break; + + case op_batch_set_bpb: + port->batch_bpb(&receive->p_batch_setbpb, sendL); + ///case op_insert: default: gds__log("SERVER/process_packet: don't understand packet type %d", receive->p_operation); diff --git a/src/yvalve/YObjects.h b/src/yvalve/YObjects.h index a33b813291..bf3c5de056 100644 --- a/src/yvalve/YObjects.h +++ b/src/yvalve/YObjects.h @@ -335,6 +335,34 @@ public: YStatement* statement; }; +class YBatch FB_FINAL : + public YHelper > +{ +public: + static const ISC_STATUS ERROR_CODE = isc_bad_result_set; // isc_bad_batch + + YBatch(YAttachment* anAttachment, Firebird::IBatch* aNext); + + void destroy(unsigned dstrFlags); + + // IBatch implementation + void add(Firebird::CheckStatusWrapper* status, unsigned count, const void* inBuffer); + void addBlob(Firebird::CheckStatusWrapper* status, unsigned length, const void* inBuffer, ISC_QUAD* blobId, + unsigned parLength, const unsigned char* par); + void appendBlobData(Firebird::CheckStatusWrapper* status, unsigned length, const void* inBuffer); + void addBlobStream(Firebird::CheckStatusWrapper* status, unsigned length, const void* inBuffer); + unsigned getBlobAlignment(Firebird::CheckStatusWrapper* status); + Firebird::IMessageMetadata* getMetadata(Firebird::CheckStatusWrapper* status); + void registerBlob(Firebird::CheckStatusWrapper* status, const ISC_QUAD* existingBlob, ISC_QUAD* blobId); + Firebird::IBatchCompletionState* execute(Firebird::CheckStatusWrapper* status, Firebird::ITransaction* transaction); + void cancel(Firebird::CheckStatusWrapper* status); + void setDefaultBpb(Firebird::CheckStatusWrapper* status, unsigned parLength, const unsigned char* par); + +public: + YAttachment* attachment; +}; + + class YMetadata { public: @@ -381,6 +409,8 @@ public: unsigned int getTimeout(Firebird::CheckStatusWrapper* status); void setTimeout(Firebird::CheckStatusWrapper* status, unsigned int timeOut); + YBatch* createBatch(Firebird::CheckStatusWrapper* status, Firebird::IMessageMetadata* inMetadata, + unsigned parLength, const unsigned char* par); public: Firebird::Mutex statementMutex; @@ -477,6 +507,9 @@ public: void setIdleTimeout(Firebird::CheckStatusWrapper* status, unsigned int timeOut); unsigned int getStatementTimeout(Firebird::CheckStatusWrapper* status); void setStatementTimeout(Firebird::CheckStatusWrapper* status, unsigned int timeOut); + YBatch* createBatch(Firebird::CheckStatusWrapper* status, Firebird::ITransaction* transaction, + unsigned stmtLength, const char* sqlStmt, unsigned dialect, + Firebird::IMessageMetadata* inMetadata, unsigned parLength, const unsigned char* par); public: Firebird::IProvider* provider; diff --git a/src/yvalve/utl.cpp b/src/yvalve/utl.cpp index 856e237788..9694c3b960 100644 --- a/src/yvalve/utl.cpp +++ b/src/yvalve/utl.cpp @@ -757,8 +757,16 @@ public: k = ClumpletReader::Tpb; tag = isc_tpb_version3; break; + case BATCH: + k = ClumpletReader::WideTagged; + tag = IBatch::VERSION1; + break; + case BPB: + k = ClumpletReader::Tagged; + tag = isc_bpb_version1; + break; default: - fatal_exception::raiseFmt("Wrong parameters block kind %d, should be from %d to %d", kind, DPB, TPB); + fatal_exception::raiseFmt("Wrong parameters block kind %d, should be from %d to %d", kind, DPB, BPB); break; } diff --git a/src/yvalve/why.cpp b/src/yvalve/why.cpp index 8fab719526..607513fc67 100644 --- a/src/yvalve/why.cpp +++ b/src/yvalve/why.cpp @@ -148,6 +148,8 @@ public: IMetadataBuilder* getBuilder(CheckStatusWrapper* status); unsigned getMessageLength(CheckStatusWrapper* status); + unsigned getAlignment(CheckStatusWrapper* status); + unsigned getAlignedLength(CheckStatusWrapper* status); void gatherData(DataBuffer& to); // Copy data from SQLDA into target buffer. void scatterData(DataBuffer& from); @@ -170,7 +172,7 @@ private: unsigned indOffset; } *offsets; - unsigned length; + unsigned length, alignment; bool speedHackEnabled; // May be user by stupid luck use right buffer format even with SQLDA interface?.. }; @@ -223,7 +225,7 @@ private: }; SQLDAMetadata::SQLDAMetadata(const XSQLDA* aSqlda) - : sqlda(aSqlda), count(0), offsets(NULL), length(0), speedHackEnabled(false) + : sqlda(aSqlda), count(0), offsets(NULL), length(0), alignment(0), speedHackEnabled(false) { if (sqlda && sqlda->version != SQLDA_VERSION1) { @@ -457,10 +459,14 @@ void SQLDAMetadata::assign() } // No matter how good or bad is the way data is placed in message buffer, it cannot be changed // because changing of it on current codebase will completely kill remote module and may be the engine as well + unsigned dtype; length = fb_utils::sqlTypeToDsc(length, var.sqltype, var.sqllen, - NULL /*dtype*/, NULL /*length*/, &it.offset, &it.indOffset); + &dtype, NULL /*length*/, &it.offset, &it.indOffset); if (it.offset != var.sqldata - base || it.indOffset != ((ISC_SCHAR*) (var.sqlind)) - base) speedHackEnabled = false; // No luck + + if (dtype < DTYPE_TYPE_MAX) + alignment = MAX(alignment, type_alignments[dtype]); } } @@ -471,6 +477,20 @@ unsigned SQLDAMetadata::getMessageLength(CheckStatusWrapper* status) return length; } +unsigned SQLDAMetadata::getAlignment(CheckStatusWrapper* status) +{ + if (!offsets) + assign(); + return alignment; +} + +unsigned SQLDAMetadata::getAlignedLength(CheckStatusWrapper* status) +{ + if (!offsets) + assign(); + return FB_ALIGN(length, alignment); +} + void SQLDAMetadata::gatherData(DataBuffer& to) { fb_assert(sqlda); // Ensure that data is gathered before take off because later they can be already changed @@ -2687,6 +2707,32 @@ ISC_STATUS API_ROUTINE fb_dsql_set_timeout(ISC_STATUS* userStatus, FB_API_HANDLE } +// Get interface by legacy handle +/*ISC_STATUS API_ROUTINE fb_get_statement_interface(ISC_STATUS* userStatus, FB_API_HANDLE* stmtHandle, + void** stmtIface) +{ + StatusVector status(userStatus); + CheckStatusWrapper statusWrapper(&status); + + try + { + RefPtr statement(translateHandle(statements, stmtHandle)); + statement->checkPrepared(); + + fb_assert(statement->statement); + IStatement* rc = statement->statement; + rc->addRef(); + *stmtIface = rc; + } + catch (const Exception& e) + { + e.stuffException(&statusWrapper); + } + + return status[1]; +} +*/ + // Provide information on sql statement. ISC_STATUS API_ROUTINE isc_dsql_sql_info(ISC_STATUS* userStatus, FB_API_HANDLE* stmtHandle, SSHORT itemLength, const SCHAR* items, SSHORT bufferLength, SCHAR* buffer) @@ -4317,7 +4363,7 @@ ITransaction* YStatement::execute(CheckStatusWrapper* status, ITransaction* tran return transaction; } -IResultSet* YStatement::openCursor(Firebird::CheckStatusWrapper* status, ITransaction* transaction, +IResultSet* YStatement::openCursor(CheckStatusWrapper* status, ITransaction* transaction, IMessageMetadata* inMetadata, void* inBuffer, IMessageMetadata* outMetadata, unsigned int flags) { try @@ -4369,6 +4415,31 @@ void YStatement::free(CheckStatusWrapper* status) } } +YBatch* YStatement::createBatch(CheckStatusWrapper* status, IMessageMetadata* inMetadata, unsigned parLength, + const unsigned char* par) +{ + try + { + YEntry entry(status, this); + + IBatch* batch = entry.next()->createBatch(status, inMetadata, parLength, par); + if (status->getState() & Firebird::IStatus::STATE_ERRORS) + { + return NULL; + } + + YBatch* newBatch = FB_NEW YBatch(attachment, batch); + newBatch->addRef(); + return newBatch; + } + catch (const Exception& e) + { + e.stuffException(status); + } + + return NULL; +} + //------------------------------------- IscStatement::~IscStatement() @@ -4756,6 +4827,177 @@ void YResultSet::close(CheckStatusWrapper* status) } } +//------------------------------------- + + +YBatch::YBatch(YAttachment* anAttachment, IBatch* aNext) + : YHelper(aNext), + attachment(anAttachment) +{ } + + +void YBatch::destroy(unsigned dstrFlags) +{ + destroy2(dstrFlags); +} + + +void YBatch::add(CheckStatusWrapper* status, unsigned count, const void* inBuffer) +{ + try + { + YEntry entry(status, this); + + entry.next()->add(status, count, inBuffer); + } + catch (const Exception& e) + { + e.stuffException(status); + } +} + + +void YBatch::addBlob(CheckStatusWrapper* status, unsigned length, const void* inBuffer, ISC_QUAD* blobId, + unsigned parLength, const unsigned char* par) +{ + try + { + YEntry entry(status, this); + + entry.next()->addBlob(status, length, inBuffer, blobId, parLength, par); + } + catch (const Exception& e) + { + e.stuffException(status); + } +} + + +void YBatch::appendBlobData(CheckStatusWrapper* status, unsigned length, const void* inBuffer) +{ + try + { + YEntry entry(status, this); + + entry.next()->appendBlobData(status, length, inBuffer); + } + catch (const Exception& e) + { + e.stuffException(status); + } +} + + +void YBatch::addBlobStream(CheckStatusWrapper* status, unsigned length, const void* inBuffer) +{ + try + { + YEntry entry(status, this); + + entry.next()->addBlobStream(status, length, inBuffer); + } + catch (const Exception& e) + { + e.stuffException(status); + } +} + + +unsigned YBatch::getBlobAlignment(CheckStatusWrapper* status) +{ + try + { + YEntry entry(status, this); + + return entry.next()->getBlobAlignment(status); + } + catch (const Exception& e) + { + e.stuffException(status); + } + + return 0; +} + + +void YBatch::setDefaultBpb(CheckStatusWrapper* status, unsigned parLength, const unsigned char* par) +{ + try + { + YEntry entry(status, this); + + entry.next()->setDefaultBpb(status, parLength, par); + } + catch (const Exception& e) + { + e.stuffException(status); + } +} + + +IMessageMetadata* YBatch::getMetadata(CheckStatusWrapper* status) +{ + try + { + YEntry entry(status, this); + + return entry.next()->getMetadata(status); + } + catch (const Exception& e) + { + e.stuffException(status); + } + + return 0; +} + + +void YBatch::registerBlob(CheckStatusWrapper* status, const ISC_QUAD* existingBlob, ISC_QUAD* blobId) +{ + try + { + YEntry entry(status, this); + + entry.next()->registerBlob(status, existingBlob, blobId); + } + catch (const Exception& e) + { + e.stuffException(status); + } +} + + +IBatchCompletionState* YBatch::execute(CheckStatusWrapper* status, ITransaction* transaction) +{ + try + { + YEntry entry(status, this); + + return entry.next()->execute(status, transaction); + } + catch (const Exception& e) + { + e.stuffException(status); + } + + return NULL; +} + + +void YBatch::cancel(CheckStatusWrapper* status) +{ + try + { + YEntry entry(status, this); + + entry.next()->cancel(status); + } + catch (const Exception& e) + { + e.stuffException(status); + } +} + //------------------------------------- @@ -5655,8 +5897,39 @@ void YAttachment::setStatementTimeout(CheckStatusWrapper* status, unsigned int t } -//------------------------------------- +YBatch* YAttachment::createBatch(CheckStatusWrapper* status, ITransaction* transaction, + unsigned stmtLength, const char* sqlStmt, unsigned dialect, + IMessageMetadata* inMetadata, unsigned parLength, const unsigned char* par) +{ + try + { + YEntry entry(status, this); + NextTransaction trans; + if (transaction) + getNextTransaction(status, transaction, trans); + + IBatch* batch = entry.next()->createBatch(status, trans, stmtLength, sqlStmt, dialect, + inMetadata, parLength, par); + if (status->getState() & Firebird::IStatus::STATE_ERRORS) + { + return NULL; + } + + YBatch* newBatch = FB_NEW YBatch(this, batch); + newBatch->addRef(); + return newBatch; + } + catch (const Exception& e) + { + e.stuffException(status); + } + + return NULL; +} + + +//------------------------------------- YService::YService(IProvider* aProvider, IService* aNext, bool utf8) : YHelper(aNext), diff --git a/src/yvalve/why_proto.h b/src/yvalve/why_proto.h index c8d659d827..31c4c1d809 100644 --- a/src/yvalve/why_proto.h +++ b/src/yvalve/why_proto.h @@ -90,6 +90,7 @@ ISC_STATUS API_ROUTINE isc_dsql_prepare_m(ISC_STATUS*, FB_API_HANDLE*, ISC_STATUS API_ROUTINE isc_dsql_set_cursor_name(ISC_STATUS*, FB_API_HANDLE*, const SCHAR*, USHORT); ISC_STATUS API_ROUTINE fb_dsql_set_timeout(ISC_STATUS*, FB_API_HANDLE*, ULONG timeout); +//ISC_STATUS API_ROUTINE fb_get_statement_interface(ISC_STATUS*, FB_API_HANDLE*, void**); ISC_STATUS API_ROUTINE isc_dsql_sql_info(ISC_STATUS*, FB_API_HANDLE*, SSHORT, const SCHAR*, SSHORT, SCHAR*); //ISC_STATUS API_ROUTINE isc_prepare_transaction2(ISC_STATUS*, FB_API_HANDLE*, USHORT,